trigger ci

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>

.github/CODEOWNERS

@@ -36,7 +36,6 @@
 /api/core/workflow/graph/ @laipz8200 @QuantumGhost
 /api/core/workflow/graph_events/ @laipz8200 @QuantumGhost
 /api/core/workflow/node_events/ @laipz8200 @QuantumGhost
-/api/graphon/model_runtime/ @laipz8200 @WH-2099
 
 # Backend - Workflow - Nodes (Agent, Iteration, Loop, LLM)
 /api/core/workflow/nodes/agent/ @Nov1c444

.github/actions/setup-web/action.yml

@@ -1,12 +1,12 @@
 name: Setup Web Environment
+description: A GitHub Action to set up the web environment using Vite+.
 
 runs:
   using: composite
   steps:
     - name: Setup Vite+
-      uses: voidzero-dev/setup-vp@20553a7a7429c429a74894104a2835d7fed28a72 # v1.3.0
+      uses: hyoban/setup-vp@96511aa421048609564ade4427c73d0078d4afc1 # v1.3.0
       with:
-        working-directory: web
         node-version-file: .nvmrc
         cache: true
         run-install: true

.github/workflows/api-tests.yml

@@ -14,18 +14,17 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test:
-    name: API Tests
+  api-unit:
+    name: API Unit Tests
     runs-on: ubuntu-latest
     env:
-      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      COVERAGE_FILE: coverage-unit
     defaults:
       run:
         shell: bash
     strategy:
       matrix:
         python-version:
-          - "3.11"
           - "3.12"
 
     steps:
@@ -36,7 +35,7 @@ jobs:
           persist-credentials: false
 
       - name: Setup UV and Python
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
         with:
           enable-cache: true
           python-version: ${{ matrix.python-version }}
@@ -51,6 +50,52 @@ jobs:
       - name: Run dify config tests
         run: uv run --project api dev/pytest/pytest_config_tests.py
 
+      - name: Run Unit Tests
+        run: uv run --project api bash dev/pytest/pytest_unit_tests.sh
+
+      - name: Upload unit coverage data
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: api-coverage-unit
+          path: coverage-unit
+          retention-days: 1
+
+  api-integration:
+    name: API Integration Tests
+    runs-on: ubuntu-latest
+    env:
+      COVERAGE_FILE: coverage-integration
+      STORAGE_TYPE: opendal
+      OPENDAL_SCHEME: fs
+      OPENDAL_FS_ROOT: /tmp/dify-storage
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        python-version:
+          - "3.12"
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Setup UV and Python
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+          cache-dependency-glob: api/uv.lock
+
+      - name: Check UV lockfile
+        run: uv lock --project api --check
+
+      - name: Install dependencies
+        run: uv sync --project api --dev
+
       - name: Set up dotenvs
         run: |
           cp docker/.env.example docker/.env
@@ -74,23 +119,91 @@ jobs:
         run: |
           cp api/tests/integration_tests/.env.example api/tests/integration_tests/.env
 
-      - name: Run API Tests
-        env:
-          STORAGE_TYPE: opendal
-          OPENDAL_SCHEME: fs
-          OPENDAL_FS_ROOT: /tmp/dify-storage
+      - name: Run Integration Tests
         run: |
           uv run --project api pytest \
             -n auto \
             --timeout "${PYTEST_TIMEOUT:-180}" \
             api/tests/integration_tests/workflow \
             api/tests/integration_tests/tools \
-            api/tests/test_containers_integration_tests \
-            api/tests/unit_tests
+            api/tests/test_containers_integration_tests
+
+      - name: Upload integration coverage data
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: api-coverage-integration
+          path: coverage-integration
+          retention-days: 1
+
+  api-coverage:
+    name: API Coverage
+    runs-on: ubuntu-latest
+    needs:
+      - api-unit
+      - api-integration
+    env:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      COVERAGE_FILE: .coverage
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Setup UV and Python
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
+        with:
+          enable-cache: true
+          python-version: "3.12"
+          cache-dependency-glob: api/uv.lock
+
+      - name: Install dependencies
+        run: uv sync --project api --dev
+
+      - name: Download coverage data
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: coverage-data
+          pattern: api-coverage-*
+          merge-multiple: true
+
+      - name: Combine coverage
+        run: |
+          set -euo pipefail
+
+          echo "### API Coverage" >> "$GITHUB_STEP_SUMMARY"
+          echo "" >> "$GITHUB_STEP_SUMMARY"
+          echo "Merged backend coverage report generated for Codecov project status." >> "$GITHUB_STEP_SUMMARY"
+          echo "" >> "$GITHUB_STEP_SUMMARY"
+
+          unit_coverage="$(find coverage-data -type f -name coverage-unit -print -quit)"
+          integration_coverage="$(find coverage-data -type f -name coverage-integration -print -quit)"
+          : "${unit_coverage:?coverage-unit artifact not found}"
+          : "${integration_coverage:?coverage-integration artifact not found}"
+
+          report_file="$(mktemp)"
+          uv run --project api coverage combine "$unit_coverage" "$integration_coverage"
+          uv run --project api coverage report --show-missing | tee "$report_file"
+          echo "Summary: \`$(tail -n 1 "$report_file")\`" >> "$GITHUB_STEP_SUMMARY"
+          {
+            echo ""
+            echo "<details><summary>Coverage report</summary>"
+            echo ""
+            echo '```'
+            cat "$report_file"
+            echo '```'
+            echo "</details>"
+          } >> "$GITHUB_STEP_SUMMARY"
+          uv run --project api coverage xml -o coverage.xml
 
       - name: Report coverage
-        if: ${{ env.CODECOV_TOKEN != '' && matrix.python-version == '3.12' }}
-        uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5.5.3
+        if: ${{ env.CODECOV_TOKEN != '' }}
+        uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
         with:
           files: ./coverage.xml
           disable_search: true

.github/workflows/autofix.yml

@@ -2,6 +2,9 @@ name: autofix.ci
 on:
   pull_request:
     branches: ["main"]
+  merge_group:
+    branches: ["main"]
+    types: [checks_requested]
   push:
     branches: ["main"]
 permissions:
@@ -12,9 +15,15 @@ jobs:
     if: github.repository == 'langgenius/dify'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Complete merge group check
+        if: github.event_name == 'merge_group'
+        run: echo "autofix.ci updates pull request branches, not merge group refs."
+
+      - if: github.event_name != 'merge_group'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
       - name: Check Docker Compose inputs
+        if: github.event_name != 'merge_group'
         id: docker-compose-changes
         uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
         with:
@@ -24,30 +33,38 @@ jobs:
             docker/docker-compose-template.yaml
             docker/docker-compose.yaml
       - name: Check web inputs
+        if: github.event_name != 'merge_group'
         id: web-changes
         uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
         with:
           files: |
             web/**
+            package.json
+            pnpm-lock.yaml
+            pnpm-workspace.yaml
+            .nvmrc
       - name: Check api inputs
+        if: github.event_name != 'merge_group'
         id: api-changes
         uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
         with:
           files: |
             api/**
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+      - if: github.event_name != 'merge_group'
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
           python-version: "3.11"
 
-      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      - if: github.event_name != 'merge_group'
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
 
       - name: Generate Docker Compose
-        if: steps.docker-compose-changes.outputs.any_changed == 'true'
+        if: github.event_name != 'merge_group' && steps.docker-compose-changes.outputs.any_changed == 'true'
         run: |
           cd docker
           ./generate_docker_compose
 
-      - if: steps.api-changes.outputs.any_changed == 'true'
+      - if: github.event_name != 'merge_group' && steps.api-changes.outputs.any_changed == 'true'
         run: |
           cd api
           uv sync --dev
@@ -59,13 +76,13 @@ jobs:
           uv run ruff format ..
 
       - name: count migration progress
-        if: steps.api-changes.outputs.any_changed == 'true'
+        if: github.event_name != 'merge_group' && steps.api-changes.outputs.any_changed == 'true'
         run: |
           cd api
           ./cnt_base.sh
 
       - name: ast-grep
-        if: steps.api-changes.outputs.any_changed == 'true'
+        if: github.event_name != 'merge_group' && steps.api-changes.outputs.any_changed == 'true'
         run: |
           # ast-grep exits 1 if no matches are found; allow idempotent runs.
           uvx --from ast-grep-cli ast-grep --pattern 'db.session.query($WHATEVER).filter($HERE)' --rewrite 'db.session.query($WHATEVER).where($HERE)' -l py --update-all || true
@@ -95,13 +112,14 @@ jobs:
           find . -name "*.py.bak" -type f -delete
 
       - name: Setup web environment
-        if: steps.web-changes.outputs.any_changed == 'true'
+        if: github.event_name != 'merge_group' && steps.web-changes.outputs.any_changed == 'true'
         uses: ./.github/actions/setup-web
 
       - name: ESLint autofix
-        if: steps.web-changes.outputs.any_changed == 'true'
+        if: github.event_name != 'merge_group' && steps.web-changes.outputs.any_changed == 'true'
         run: |
           cd web
           vp exec eslint --concurrency=2 --prune-suppressions --quiet || true
 
-      - uses: autofix-ci/action@7a166d7532b277f34e16238930461bf77f9d7ed8 # v1.3.3
+      - if: github.event_name != 'merge_group'
+        uses: autofix-ci/action@7a166d7532b277f34e16238930461bf77f9d7ed8 # v1.3.3

.github/workflows/build-push.yml

@@ -24,27 +24,39 @@ env:
 
 jobs:
   build:
-    runs-on: ${{ matrix.platform == 'linux/arm64' && 'arm64_runner' || 'ubuntu-latest' }}
+    runs-on: ${{ matrix.runs_on }}
     if: github.repository == 'langgenius/dify'
     strategy:
       matrix:
         include:
           - service_name: "build-api-amd64"
             image_name_env: "DIFY_API_IMAGE_NAME"
-            context: "api"
+            artifact_context: "api"
+            build_context: "{{defaultContext}}:api"
+            file: "Dockerfile"
             platform: linux/amd64
+            runs_on: ubuntu-latest
           - service_name: "build-api-arm64"
             image_name_env: "DIFY_API_IMAGE_NAME"
-            context: "api"
+            artifact_context: "api"
+            build_context: "{{defaultContext}}:api"
+            file: "Dockerfile"
             platform: linux/arm64
+            runs_on: ubuntu-24.04-arm
           - service_name: "build-web-amd64"
             image_name_env: "DIFY_WEB_IMAGE_NAME"
-            context: "web"
+            artifact_context: "web"
+            build_context: "{{defaultContext}}"
+            file: "web/Dockerfile"
             platform: linux/amd64
+            runs_on: ubuntu-latest
           - service_name: "build-web-arm64"
             image_name_env: "DIFY_WEB_IMAGE_NAME"
-            context: "web"
+            artifact_context: "web"
+            build_context: "{{defaultContext}}"
+            file: "web/Dockerfile"
             platform: linux/arm64
+            runs_on: ubuntu-24.04-arm
 
     steps:
       - name: Prepare
@@ -58,9 +70,6 @@ jobs:
           username: ${{ env.DOCKERHUB_USER }}
           password: ${{ env.DOCKERHUB_TOKEN }}
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
 
@@ -74,7 +83,8 @@ jobs:
         id: build
         uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
         with:
-          context: "{{defaultContext}}:${{ matrix.context }}"
+          context: ${{ matrix.build_context }}
+          file: ${{ matrix.file }}
           platforms: ${{ matrix.platform }}
           build-args: COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
           labels: ${{ steps.meta.outputs.labels }}
@@ -93,7 +103,7 @@ jobs:
       - name: Upload digest
         uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
-          name: digests-${{ matrix.context }}-${{ env.PLATFORM_PAIR }}
+          name: digests-${{ matrix.artifact_context }}-${{ env.PLATFORM_PAIR }}
           path: /tmp/digests/*
           if-no-files-found: error
           retention-days: 1

.github/workflows/db-migration-test.yml

@@ -19,7 +19,7 @@ jobs:
           persist-credentials: false
 
       - name: Setup UV and Python
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
         with:
           enable-cache: true
           python-version: "3.12"
@@ -69,7 +69,7 @@ jobs:
           persist-credentials: false
 
       - name: Setup UV and Python
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
         with:
           enable-cache: true
           python-version: "3.12"

.github/workflows/docker-build.yml

@@ -6,7 +6,12 @@ on:
       - "main"
     paths:
       - api/Dockerfile
+      - web/docker/**
       - web/Dockerfile
+      - package.json
+      - pnpm-lock.yaml
+      - pnpm-workspace.yaml
+      - .nvmrc
 
 concurrency:
   group: docker-build-${{ github.head_ref || github.run_id }}
@@ -14,26 +19,31 @@ concurrency:
 
 jobs:
   build-docker:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runs_on }}
     strategy:
       matrix:
         include:
           - service_name: "api-amd64"
             platform: linux/amd64
-            context: "api"
+            runs_on: ubuntu-latest
+            context: "{{defaultContext}}:api"
+            file: "Dockerfile"
           - service_name: "api-arm64"
             platform: linux/arm64
-            context: "api"
+            runs_on: ubuntu-24.04-arm
+            context: "{{defaultContext}}:api"
+            file: "Dockerfile"
           - service_name: "web-amd64"
             platform: linux/amd64
-            context: "web"
+            runs_on: ubuntu-latest
+            context: "{{defaultContext}}"
+            file: "web/Dockerfile"
           - service_name: "web-arm64"
             platform: linux/arm64
-            context: "web"
+            runs_on: ubuntu-24.04-arm
+            context: "{{defaultContext}}"
+            file: "web/Dockerfile"
     steps:
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
 
@@ -41,8 +51,8 @@ jobs:
         uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
         with:
           push: false
-          context: "{{defaultContext}}:${{ matrix.context }}"
-          file: "${{ matrix.file }}"
+          context: ${{ matrix.context }}
+          file: ${{ matrix.file }}
           platforms: ${{ matrix.platform }}
           cache-from: type=gha
           cache-to: type=gha,mode=max

.github/workflows/main-ci.yml

@@ -3,10 +3,14 @@ name: Main CI Pipeline
 on:
   pull_request:
     branches: ["main"]
+  merge_group:
+    branches: ["main"]
+    types: [checks_requested]
   push:
     branches: ["main"]
 
 permissions:
+  actions: write
   contents: write
   pull-requests: write
   checks: write
@@ -17,12 +21,28 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  pre_job:
+    name: Skip Duplicate Checks
+    runs-on: ubuntu-latest
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip || 'false' }}
+    steps:
+      - id: skip_check
+        continue-on-error: true
+        uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5.3.1
+        with:
+          cancel_others: 'true'
+          concurrent_skipping: same_content_newer
+
   # Check which paths were changed to determine which tests to run
   check-changes:
     name: Check Changed Files
+    needs: pre_job
+    if: needs.pre_job.outputs.should_skip != 'true'
     runs-on: ubuntu-latest
     outputs:
       api-changed: ${{ steps.changes.outputs.api }}
+      e2e-changed: ${{ steps.changes.outputs.e2e }}
       web-changed: ${{ steps.changes.outputs.web }}
       vdb-changed: ${{ steps.changes.outputs.vdb }}
       migration-changed: ${{ steps.changes.outputs.migration }}
@@ -34,49 +54,372 @@ jobs:
           filters: |
             api:
               - 'api/**'
-              - 'docker/**'
               - '.github/workflows/api-tests.yml'
+              - '.github/workflows/expose_service_ports.sh'
+              - 'docker/.env.example'
+              - 'docker/middleware.env.example'
+              - 'docker/docker-compose.middleware.yaml'
+              - 'docker/docker-compose-template.yaml'
+              - 'docker/generate_docker_compose'
+              - 'docker/ssrf_proxy/**'
+              - 'docker/volumes/sandbox/conf/**'
             web:
               - 'web/**'
+              - 'package.json'
+              - 'pnpm-lock.yaml'
+              - 'pnpm-workspace.yaml'
+              - '.nvmrc'
               - '.github/workflows/web-tests.yml'
               - '.github/actions/setup-web/**'
+            e2e:
+              - 'api/**'
+              - 'api/pyproject.toml'
+              - 'api/uv.lock'
+              - 'e2e/**'
+              - 'web/**'
+              - 'package.json'
+              - 'pnpm-lock.yaml'
+              - 'pnpm-workspace.yaml'
+              - '.nvmrc'
+              - 'docker/docker-compose.middleware.yaml'
+              - 'docker/middleware.env.example'
+              - '.github/workflows/web-e2e.yml'
+              - '.github/actions/setup-web/**'
             vdb:
               - 'api/core/rag/datasource/**'
-              - 'docker/**'
+              - 'api/tests/integration_tests/vdb/**'
               - '.github/workflows/vdb-tests.yml'
+              - '.github/workflows/expose_service_ports.sh'
+              - 'docker/.env.example'
+              - 'docker/middleware.env.example'
+              - 'docker/docker-compose.yaml'
+              - 'docker/docker-compose-template.yaml'
+              - 'docker/generate_docker_compose'
+              - 'docker/certbot/**'
+              - 'docker/couchbase-server/**'
+              - 'docker/elasticsearch/**'
+              - 'docker/iris/**'
+              - 'docker/nginx/**'
+              - 'docker/pgvector/**'
+              - 'docker/ssrf_proxy/**'
+              - 'docker/startupscripts/**'
+              - 'docker/tidb/**'
+              - 'docker/volumes/**'
               - 'api/uv.lock'
               - 'api/pyproject.toml'
             migration:
               - 'api/migrations/**'
+              - 'api/.env.example'
               - '.github/workflows/db-migration-test.yml'
+              - '.github/workflows/expose_service_ports.sh'
+              - 'docker/.env.example'
+              - 'docker/middleware.env.example'
+              - 'docker/docker-compose.middleware.yaml'
+              - 'docker/docker-compose-template.yaml'
+              - 'docker/generate_docker_compose'
+              - 'docker/ssrf_proxy/**'
+              - 'docker/volumes/sandbox/conf/**'
 
-  # Run tests in parallel
-  api-tests:
-    name: API Tests
-    needs: check-changes
-    if: needs.check-changes.outputs.api-changed == 'true'
+  # Run tests in parallel while always emitting stable required checks.
+  api-tests-run:
+    name: Run API Tests
+    needs:
+      - pre_job
+      - check-changes
+    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.api-changed == 'true'
     uses: ./.github/workflows/api-tests.yml
     secrets: inherit
 
-  web-tests:
-    name: Web Tests
-    needs: check-changes
-    if: needs.check-changes.outputs.web-changed == 'true'
+  api-tests-skip:
+    name: Skip API Tests
+    needs:
+      - pre_job
+      - check-changes
+    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.api-changed != 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Report skipped API tests
+        run: echo "No API-related changes detected; skipping API tests."
+
+  api-tests:
+    name: API Tests
+    if: ${{ always() }}
+    needs:
+      - pre_job
+      - check-changes
+      - api-tests-run
+      - api-tests-skip
+    runs-on: ubuntu-latest
+    steps:
+      - name: Finalize API Tests status
+        env:
+          SHOULD_SKIP_WORKFLOW: ${{ needs.pre_job.outputs.should_skip }}
+          TESTS_CHANGED: ${{ needs.check-changes.outputs.api-changed }}
+          RUN_RESULT: ${{ needs.api-tests-run.result }}
+          SKIP_RESULT: ${{ needs.api-tests-skip.result }}
+        run: |
+          if [[ "$SHOULD_SKIP_WORKFLOW" == 'true' ]]; then
+            echo "API tests were skipped because this workflow run duplicated a successful or newer run."
+            exit 0
+          fi
+
+          if [[ "$TESTS_CHANGED" == 'true' ]]; then
+            if [[ "$RUN_RESULT" == 'success' ]]; then
+              echo "API tests ran successfully."
+              exit 0
+            fi
+
+            echo "API tests were required but finished with result: $RUN_RESULT" >&2
+            exit 1
+          fi
+
+          if [[ "$SKIP_RESULT" == 'success' ]]; then
+            echo "API tests were skipped because no API-related files changed."
+            exit 0
+          fi
+
+          echo "API tests were not required, but the skip job finished with result: $SKIP_RESULT" >&2
+          exit 1
+
+  web-tests-run:
+    name: Run Web Tests
+    needs:
+      - pre_job
+      - check-changes
+    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.web-changed == 'true'
     uses: ./.github/workflows/web-tests.yml
     secrets: inherit
 
+  web-tests-skip:
+    name: Skip Web Tests
+    needs:
+      - pre_job
+      - check-changes
+    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.web-changed != 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Report skipped web tests
+        run: echo "No web-related changes detected; skipping web tests."
+
+  web-tests:
+    name: Web Tests
+    if: ${{ always() }}
+    needs:
+      - pre_job
+      - check-changes
+      - web-tests-run
+      - web-tests-skip
+    runs-on: ubuntu-latest
+    steps:
+      - name: Finalize Web Tests status
+        env:
+          SHOULD_SKIP_WORKFLOW: ${{ needs.pre_job.outputs.should_skip }}
+          TESTS_CHANGED: ${{ needs.check-changes.outputs.web-changed }}
+          RUN_RESULT: ${{ needs.web-tests-run.result }}
+          SKIP_RESULT: ${{ needs.web-tests-skip.result }}
+        run: |
+          if [[ "$SHOULD_SKIP_WORKFLOW" == 'true' ]]; then
+            echo "Web tests were skipped because this workflow run duplicated a successful or newer run."
+            exit 0
+          fi
+
+          if [[ "$TESTS_CHANGED" == 'true' ]]; then
+            if [[ "$RUN_RESULT" == 'success' ]]; then
+              echo "Web tests ran successfully."
+              exit 0
+            fi
+
+            echo "Web tests were required but finished with result: $RUN_RESULT" >&2
+            exit 1
+          fi
+
+          if [[ "$SKIP_RESULT" == 'success' ]]; then
+            echo "Web tests were skipped because no web-related files changed."
+            exit 0
+          fi
+
+          echo "Web tests were not required, but the skip job finished with result: $SKIP_RESULT" >&2
+          exit 1
+
+  web-e2e-run:
+    name: Run Web Full-Stack E2E
+    needs:
+      - pre_job
+      - check-changes
+    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.e2e-changed == 'true'
+    uses: ./.github/workflows/web-e2e.yml
+
+  web-e2e-skip:
+    name: Skip Web Full-Stack E2E
+    needs:
+      - pre_job
+      - check-changes
+    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.e2e-changed != 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Report skipped web full-stack e2e
+        run: echo "No E2E-related changes detected; skipping web full-stack E2E."
+
+  web-e2e:
+    name: Web Full-Stack E2E
+    if: ${{ always() }}
+    needs:
+      - pre_job
+      - check-changes
+      - web-e2e-run
+      - web-e2e-skip
+    runs-on: ubuntu-latest
+    steps:
+      - name: Finalize Web Full-Stack E2E status
+        env:
+          SHOULD_SKIP_WORKFLOW: ${{ needs.pre_job.outputs.should_skip }}
+          TESTS_CHANGED: ${{ needs.check-changes.outputs.e2e-changed }}
+          RUN_RESULT: ${{ needs.web-e2e-run.result }}
+          SKIP_RESULT: ${{ needs.web-e2e-skip.result }}
+        run: |
+          if [[ "$SHOULD_SKIP_WORKFLOW" == 'true' ]]; then
+            echo "Web full-stack E2E was skipped because this workflow run duplicated a successful or newer run."
+            exit 0
+          fi
+
+          if [[ "$TESTS_CHANGED" == 'true' ]]; then
+            if [[ "$RUN_RESULT" == 'success' ]]; then
+              echo "Web full-stack E2E ran successfully."
+              exit 0
+            fi
+
+            echo "Web full-stack E2E was required but finished with result: $RUN_RESULT" >&2
+            exit 1
+          fi
+
+          if [[ "$SKIP_RESULT" == 'success' ]]; then
+            echo "Web full-stack E2E was skipped because no E2E-related files changed."
+            exit 0
+          fi
+
+          echo "Web full-stack E2E was not required, but the skip job finished with result: $SKIP_RESULT" >&2
+          exit 1
+
   style-check:
     name: Style Check
+    needs: pre_job
+    if: needs.pre_job.outputs.should_skip != 'true'
     uses: ./.github/workflows/style.yml
 
+  vdb-tests-run:
+    name: Run VDB Tests
+    needs:
+      - pre_job
+      - check-changes
+    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.vdb-changed == 'true'
+    uses: ./.github/workflows/vdb-tests.yml
+
+  vdb-tests-skip:
+    name: Skip VDB Tests
+    needs:
+      - pre_job
+      - check-changes
+    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.vdb-changed != 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Report skipped VDB tests
+        run: echo "No VDB-related changes detected; skipping VDB tests."
+
   vdb-tests:
     name: VDB Tests
-    needs: check-changes
-    if: needs.check-changes.outputs.vdb-changed == 'true'
-    uses: ./.github/workflows/vdb-tests.yml
+    if: ${{ always() }}
+    needs:
+      - pre_job
+      - check-changes
+      - vdb-tests-run
+      - vdb-tests-skip
+    runs-on: ubuntu-latest
+    steps:
+      - name: Finalize VDB Tests status
+        env:
+          SHOULD_SKIP_WORKFLOW: ${{ needs.pre_job.outputs.should_skip }}
+          TESTS_CHANGED: ${{ needs.check-changes.outputs.vdb-changed }}
+          RUN_RESULT: ${{ needs.vdb-tests-run.result }}
+          SKIP_RESULT: ${{ needs.vdb-tests-skip.result }}
+        run: |
+          if [[ "$SHOULD_SKIP_WORKFLOW" == 'true' ]]; then
+            echo "VDB tests were skipped because this workflow run duplicated a successful or newer run."
+            exit 0
+          fi
+
+          if [[ "$TESTS_CHANGED" == 'true' ]]; then
+            if [[ "$RUN_RESULT" == 'success' ]]; then
+              echo "VDB tests ran successfully."
+              exit 0
+            fi
+
+            echo "VDB tests were required but finished with result: $RUN_RESULT" >&2
+            exit 1
+          fi
+
+          if [[ "$SKIP_RESULT" == 'success' ]]; then
+            echo "VDB tests were skipped because no VDB-related files changed."
+            exit 0
+          fi
+
+          echo "VDB tests were not required, but the skip job finished with result: $SKIP_RESULT" >&2
+          exit 1
+
+  db-migration-test-run:
+    name: Run DB Migration Test
+    needs:
+      - pre_job
+      - check-changes
+    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.migration-changed == 'true'
+    uses: ./.github/workflows/db-migration-test.yml
+
+  db-migration-test-skip:
+    name: Skip DB Migration Test
+    needs:
+      - pre_job
+      - check-changes
+    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.migration-changed != 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Report skipped DB migration tests
+        run: echo "No migration-related changes detected; skipping DB migration tests."
 
   db-migration-test:
     name: DB Migration Test
-    needs: check-changes
-    if: needs.check-changes.outputs.migration-changed == 'true'
-    uses: ./.github/workflows/db-migration-test.yml
+    if: ${{ always() }}
+    needs:
+      - pre_job
+      - check-changes
+      - db-migration-test-run
+      - db-migration-test-skip
+    runs-on: ubuntu-latest
+    steps:
+      - name: Finalize DB Migration Test status
+        env:
+          SHOULD_SKIP_WORKFLOW: ${{ needs.pre_job.outputs.should_skip }}
+          TESTS_CHANGED: ${{ needs.check-changes.outputs.migration-changed }}
+          RUN_RESULT: ${{ needs.db-migration-test-run.result }}
+          SKIP_RESULT: ${{ needs.db-migration-test-skip.result }}
+        run: |
+          if [[ "$SHOULD_SKIP_WORKFLOW" == 'true' ]]; then
+            echo "DB migration tests were skipped because this workflow run duplicated a successful or newer run."
+            exit 0
+          fi
+
+          if [[ "$TESTS_CHANGED" == 'true' ]]; then
+            if [[ "$RUN_RESULT" == 'success' ]]; then
+              echo "DB migration tests ran successfully."
+              exit 0
+            fi
+
+            echo "DB migration tests were required but finished with result: $RUN_RESULT" >&2
+            exit 1
+          fi
+
+          if [[ "$SKIP_RESULT" == 'success' ]]; then
+            echo "DB migration tests were skipped because no migration-related files changed."
+            exit 0
+          fi
+
+          echo "DB migration tests were not required, but the skip job finished with result: $SKIP_RESULT" >&2
+          exit 1

.github/workflows/pyrefly-diff.yml

@@ -22,7 +22,7 @@ jobs:
           fetch-depth: 0
 
       - name: Setup Python & UV
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
         with:
           enable-cache: true
 
@@ -50,6 +50,17 @@ jobs:
         run: |
           diff -u /tmp/pyrefly_base.txt /tmp/pyrefly_pr.txt > pyrefly_diff.txt || true
 
+      - name: Check if line counts match
+        id: line_count_check
+        run: |
+          base_lines=$(wc -l < /tmp/pyrefly_base.txt)
+          pr_lines=$(wc -l < /tmp/pyrefly_pr.txt)
+          if [ "$base_lines" -eq "$pr_lines" ]; then
+            echo "same=true" >> $GITHUB_OUTPUT
+          else
+            echo "same=false" >> $GITHUB_OUTPUT
+          fi
+
       - name: Save PR number
         run: |
           echo ${{ github.event.pull_request.number }} > pr_number.txt
@@ -63,7 +74,7 @@ jobs:
             pr_number.txt
 
       - name: Comment PR with pyrefly diff
-        if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
+        if: ${{ github.event.pull_request.head.repo.full_name == github.repository && steps.line_count_check.outputs.same == 'false' }}
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/semantic-pull-request.yml

@@ -7,6 +7,9 @@ on:
       - edited
       - reopened
       - synchronize
+  merge_group:
+    branches: ["main"]
+    types: [checks_requested]
 
 jobs:
   lint:
@@ -15,7 +18,11 @@ jobs:
       pull-requests: read
     runs-on: ubuntu-latest
     steps:
+      - name: Complete merge group check
+        if: github.event_name == 'merge_group'
+        run: echo "Semantic PR title validation is handled on pull requests."
       - name: Check title
+        if: github.event_name == 'pull_request'
         uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/style.yml

@@ -33,7 +33,7 @@ jobs:
 
       - name: Setup UV and Python
         if: steps.changed-files.outputs.any_changed == 'true'
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
         with:
           enable-cache: false
           python-version: "3.12"
@@ -49,7 +49,7 @@ jobs:
 
       - name: Run Type Checks
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: make type-check
+        run: make type-check-core
 
       - name: Dotenv check
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -77,6 +77,10 @@ jobs:
         with:
           files: |
             web/**
+            package.json
+            pnpm-lock.yaml
+            pnpm-workspace.yaml
+            .nvmrc
             .github/workflows/style.yml
             .github/actions/setup-web/**
 
@@ -90,9 +94,9 @@ jobs:
         uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
         with:
           path: web/.eslintcache
-          key: ${{ runner.os }}-web-eslint-${{ hashFiles('web/package.json', 'web/pnpm-lock.yaml', 'web/eslint.config.mjs', 'web/eslint.constants.mjs', 'web/plugins/eslint/**') }}-${{ github.sha }}
+          key: ${{ runner.os }}-web-eslint-${{ hashFiles('web/package.json', 'pnpm-lock.yaml', 'web/eslint.config.mjs', 'web/eslint.constants.mjs', 'web/plugins/eslint/**') }}-${{ github.sha }}
           restore-keys: |
-            ${{ runner.os }}-web-eslint-${{ hashFiles('web/package.json', 'web/pnpm-lock.yaml', 'web/eslint.config.mjs', 'web/eslint.constants.mjs', 'web/plugins/eslint/**') }}-
+            ${{ runner.os }}-web-eslint-${{ hashFiles('web/package.json', 'pnpm-lock.yaml', 'web/eslint.config.mjs', 'web/eslint.constants.mjs', 'web/plugins/eslint/**') }}-
 
       - name: Web style check
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -102,12 +106,12 @@ jobs:
       - name: Web tsslint
         if: steps.changed-files.outputs.any_changed == 'true'
         working-directory: ./web
-        run: vp run lint:tss
+        run: vp run lint:tss --cache
 
       - name: Web type check
         if: steps.changed-files.outputs.any_changed == 'true'
         working-directory: ./web
-        run: vp run type-check
+        run: vp run type-check --cache
 
       - name: Web dead code check
         if: steps.changed-files.outputs.any_changed == 'true'

.github/workflows/tool-test-sdks.yaml

@@ -6,6 +6,9 @@ on:
       - main
     paths:
       - sdks/**
+      - package.json
+      - pnpm-lock.yaml
+      - pnpm-workspace.yaml
 
 concurrency:
   group: sdk-tests-${{ github.head_ref || github.run_id }}

.github/workflows/translate-i18n-claude.yml

@@ -1,26 +1,24 @@
 name: Translate i18n Files with Claude Code
 
-# Note: claude-code-action doesn't support push events directly.
-# Push events are handled by trigger-i18n-sync.yml which sends repository_dispatch.
-# See: https://github.com/langgenius/dify/issues/30743
-
 on:
-  repository_dispatch:
-    types: [i18n-sync]
+  push:
+    branches: [main]
+    paths:
+      - 'web/i18n/en-US/*.json'
   workflow_dispatch:
     inputs:
       files:
-        description: 'Specific files to translate (space-separated, e.g., "app common"). Leave empty for all files.'
+        description: 'Specific files to translate (space-separated, e.g., "app common"). Required for full mode; leave empty in incremental mode to use en-US files changed since HEAD~1.'
         required: false
         type: string
       languages:
-        description: 'Specific languages to translate (space-separated, e.g., "zh-Hans ja-JP"). Leave empty for all supported languages.'
+        description: 'Specific languages to translate (space-separated, e.g., "zh-Hans ja-JP"). Leave empty for all supported target languages except en-US.'
         required: false
         type: string
       mode:
-        description: 'Sync mode: incremental (only changes) or full (re-check all keys)'
+        description: 'Sync mode: incremental (compare with previous en-US revision) or full (sync all keys in scope)'
         required: false
-        default: 'incremental'
+        default: incremental
         type: choice
         options:
           - incremental
@@ -30,11 +28,15 @@ permissions:
   contents: write
   pull-requests: write
 
+concurrency:
+  group: translate-i18n-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'push' }}
+
 jobs:
   translate:
     if: github.repository == 'langgenius/dify'
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    timeout-minutes: 120
 
     steps:
       - name: Checkout repository
@@ -51,380 +53,161 @@ jobs:
       - name: Setup web environment
         uses: ./.github/actions/setup-web
 
-      - name: Detect changed files and generate diff
-        id: detect_changes
+      - name: Prepare sync context
+        id: context
+        shell: bash
         run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            # Manual trigger
-            if [ -n "${{ github.event.inputs.files }}" ]; then
-              echo "CHANGED_FILES=${{ github.event.inputs.files }}" >> $GITHUB_OUTPUT
-            else
-              # Get all JSON files in en-US directory
-              files=$(ls web/i18n/en-US/*.json 2>/dev/null | xargs -n1 basename | sed 's/.json$//' | tr '\n' ' ')
-              echo "CHANGED_FILES=$files" >> $GITHUB_OUTPUT
-            fi
-            echo "TARGET_LANGS=${{ github.event.inputs.languages }}" >> $GITHUB_OUTPUT
-            echo "SYNC_MODE=${{ github.event.inputs.mode || 'incremental' }}" >> $GITHUB_OUTPUT
+          DEFAULT_TARGET_LANGS=$(awk "
+            /value: '/ {
+              value=\$2
+              gsub(/[',]/, \"\", value)
+            }
+            /supported: true/ && value != \"en-US\" {
+              printf \"%s \", value
+            }
+          " web/i18n-config/languages.ts | sed 's/[[:space:]]*$//')
 
-            # For manual trigger with incremental mode, get diff from last commit
-            # For full mode, we'll do a complete check anyway
-            if [ "${{ github.event.inputs.mode }}" == "full" ]; then
-              echo "Full mode: will check all keys" > /tmp/i18n-diff.txt
-              echo "DIFF_AVAILABLE=false" >> $GITHUB_OUTPUT
+          if [ "${{ github.event_name }}" = "push" ]; then
+            BASE_SHA="${{ github.event.before }}"
+            if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" = "0000000000000000000000000000000000000000" ]; then
+              BASE_SHA=$(git rev-parse HEAD~1 2>/dev/null || true)
+            fi
+            HEAD_SHA="${{ github.sha }}"
+            if [ -n "$BASE_SHA" ]; then
+              CHANGED_FILES=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" -- 'web/i18n/en-US/*.json' 2>/dev/null | sed -n 's@^.*/@@p' | sed 's/\.json$//' | tr '\n' ' ' | sed 's/[[:space:]]*$//')
             else
-              git diff HEAD~1..HEAD -- 'web/i18n/en-US/*.json' > /tmp/i18n-diff.txt 2>/dev/null || echo "" > /tmp/i18n-diff.txt
-              if [ -s /tmp/i18n-diff.txt ]; then
-                echo "DIFF_AVAILABLE=true" >> $GITHUB_OUTPUT
-              else
-                echo "DIFF_AVAILABLE=false" >> $GITHUB_OUTPUT
-              fi
-            fi
-          elif [ "${{ github.event_name }}" == "repository_dispatch" ]; then
-            # Triggered by push via trigger-i18n-sync.yml workflow
-            # Validate required payload fields
-            if [ -z "${{ github.event.client_payload.changed_files }}" ]; then
-              echo "Error: repository_dispatch payload missing required 'changed_files' field" >&2
-              exit 1
-            fi
-            echo "CHANGED_FILES=${{ github.event.client_payload.changed_files }}" >> $GITHUB_OUTPUT
-            echo "TARGET_LANGS=" >> $GITHUB_OUTPUT
-            echo "SYNC_MODE=${{ github.event.client_payload.sync_mode || 'incremental' }}" >> $GITHUB_OUTPUT
-
-            # Decode the base64-encoded diff from the trigger workflow
-            if [ -n "${{ github.event.client_payload.diff_base64 }}" ]; then
-              if ! echo "${{ github.event.client_payload.diff_base64 }}" | base64 -d > /tmp/i18n-diff.txt 2>&1; then
-                echo "Warning: Failed to decode base64 diff payload" >&2
-                echo "" > /tmp/i18n-diff.txt
-                echo "DIFF_AVAILABLE=false" >> $GITHUB_OUTPUT
-              elif [ -s /tmp/i18n-diff.txt ]; then
-                echo "DIFF_AVAILABLE=true" >> $GITHUB_OUTPUT
-              else
-                echo "DIFF_AVAILABLE=false" >> $GITHUB_OUTPUT
-              fi
-            else
-              echo "" > /tmp/i18n-diff.txt
-              echo "DIFF_AVAILABLE=false" >> $GITHUB_OUTPUT
+              CHANGED_FILES=$(find web/i18n/en-US -maxdepth 1 -type f -name '*.json' -print | sed -n 's@^.*/@@p' | sed 's/\.json$//' | sort | tr '\n' ' ' | sed 's/[[:space:]]*$//')
             fi
+            TARGET_LANGS="$DEFAULT_TARGET_LANGS"
+            SYNC_MODE="incremental"
           else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
+            BASE_SHA=""
+            HEAD_SHA=$(git rev-parse HEAD)
+            if [ -n "${{ github.event.inputs.languages }}" ]; then
+              TARGET_LANGS="${{ github.event.inputs.languages }}"
+            else
+              TARGET_LANGS="$DEFAULT_TARGET_LANGS"
+            fi
+            SYNC_MODE="${{ github.event.inputs.mode || 'incremental' }}"
+            if [ -n "${{ github.event.inputs.files }}" ]; then
+              CHANGED_FILES="${{ github.event.inputs.files }}"
+            elif [ "$SYNC_MODE" = "incremental" ]; then
+              BASE_SHA=$(git rev-parse HEAD~1 2>/dev/null || true)
+              if [ -n "$BASE_SHA" ]; then
+                CHANGED_FILES=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" -- 'web/i18n/en-US/*.json' 2>/dev/null | sed -n 's@^.*/@@p' | sed 's/\.json$//' | tr '\n' ' ' | sed 's/[[:space:]]*$//')
+              else
+                CHANGED_FILES=$(find web/i18n/en-US -maxdepth 1 -type f -name '*.json' -print | sed -n 's@^.*/@@p' | sed 's/\.json$//' | sort | tr '\n' ' ' | sed 's/[[:space:]]*$//')
+              fi
+            elif [ "$SYNC_MODE" = "full" ]; then
+              echo "workflow_dispatch full mode requires the files input to stay within CI limits." >&2
+              exit 1
+            else
+              CHANGED_FILES=""
+            fi
           fi
 
-          # Truncate diff if too large (keep first 50KB)
-          if [ -f /tmp/i18n-diff.txt ]; then
-            head -c 50000 /tmp/i18n-diff.txt > /tmp/i18n-diff-truncated.txt
-            mv /tmp/i18n-diff-truncated.txt /tmp/i18n-diff.txt
+          FILE_ARGS=""
+          if [ -n "$CHANGED_FILES" ]; then
+            FILE_ARGS="--file $CHANGED_FILES"
           fi
 
-          echo "Detected files: $(cat $GITHUB_OUTPUT | grep CHANGED_FILES || echo 'none')"
+          LANG_ARGS=""
+          if [ -n "$TARGET_LANGS" ]; then
+            LANG_ARGS="--lang $TARGET_LANGS"
+          fi
+
+          {
+            echo "DEFAULT_TARGET_LANGS=$DEFAULT_TARGET_LANGS"
+            echo "BASE_SHA=$BASE_SHA"
+            echo "HEAD_SHA=$HEAD_SHA"
+            echo "CHANGED_FILES=$CHANGED_FILES"
+            echo "TARGET_LANGS=$TARGET_LANGS"
+            echo "SYNC_MODE=$SYNC_MODE"
+            echo "FILE_ARGS=$FILE_ARGS"
+            echo "LANG_ARGS=$LANG_ARGS"
+          } >> "$GITHUB_OUTPUT"
+
+          echo "Files: ${CHANGED_FILES:-<none>}"
+          echo "Languages: ${TARGET_LANGS:-<none>}"
+          echo "Mode: $SYNC_MODE"
 
       - name: Run Claude Code for Translation Sync
-        if: steps.detect_changes.outputs.CHANGED_FILES != ''
-        uses: anthropics/claude-code-action@ff9acae5886d41a99ed4ec14b7dc147d55834722 # v1.0.77
+        if: steps.context.outputs.CHANGED_FILES != ''
+        uses: anthropics/claude-code-action@88c168b39e7e64da0286d812b6e9fbebb6708185 # v1.0.82
         with:
           anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          # Allow github-actions bot to trigger this workflow via repository_dispatch
-          # See: https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
           allowed_bots: 'github-actions[bot]'
+          show_full_output: ${{ github.event_name == 'workflow_dispatch' }}
           prompt: |
-            You are a professional i18n synchronization engineer for the Dify project.
-            Your task is to keep all language translations in sync with the English source (en-US).
+            You are the i18n sync agent for the Dify repository.
+            Your job is to keep translations synchronized with the English source files under `${{ github.workspace }}/web/i18n/en-US/`, then open a PR with the result.
 
-            ## CRITICAL TOOL RESTRICTIONS
-            - Use **Read** tool to read files (NOT cat or bash)
-            - Use **Edit** tool to modify JSON files (NOT node, jq, or bash scripts)
-            - Use **Bash** ONLY for: git commands, gh commands, pnpm commands
-            - Run bash commands ONE BY ONE, never combine with && or ||
-            - NEVER use `$()` command substitution - it's not supported. Split into separate commands instead.
+            Use absolute paths at all times:
+            - Repo root: `${{ github.workspace }}`
+            - Web directory: `${{ github.workspace }}/web`
+            - Language config: `${{ github.workspace }}/web/i18n-config/languages.ts`
 
-            ## WORKING DIRECTORY & ABSOLUTE PATHS
-            Claude Code sandbox working directory may vary. Always use absolute paths:
-            - For pnpm: `pnpm --dir ${{ github.workspace }}/web <command>`
-            - For git: `git -C ${{ github.workspace }} <command>`
-            - For gh: `gh --repo ${{ github.repository }} <command>`
-            - For file paths: `${{ github.workspace }}/web/i18n/`
+            Inputs:
+            - Files in scope: `${{ steps.context.outputs.CHANGED_FILES }}`
+            - Target languages: `${{ steps.context.outputs.TARGET_LANGS }}`
+            - Sync mode: `${{ steps.context.outputs.SYNC_MODE }}`
+            - Base SHA: `${{ steps.context.outputs.BASE_SHA }}`
+            - Head SHA: `${{ steps.context.outputs.HEAD_SHA }}`
+            - Scoped file args: `${{ steps.context.outputs.FILE_ARGS }}`
+            - Scoped language args: `${{ steps.context.outputs.LANG_ARGS }}`
 
-            ## EFFICIENCY RULES
-            - **ONE Edit per language file** - batch all key additions into a single Edit
-            - Insert new keys at the beginning of JSON (after `{`), lint:fix will sort them
-            - Translate ALL keys for a language mentally first, then do ONE Edit
-
-            ## Context
-            - Changed/target files: ${{ steps.detect_changes.outputs.CHANGED_FILES }}
-            - Target languages (empty means all supported): ${{ steps.detect_changes.outputs.TARGET_LANGS }}
-            - Sync mode: ${{ steps.detect_changes.outputs.SYNC_MODE }}
-            - Translation files are located in: ${{ github.workspace }}/web/i18n/{locale}/{filename}.json
-            - Language configuration is in: ${{ github.workspace }}/web/i18n-config/languages.ts
-            - Git diff is available: ${{ steps.detect_changes.outputs.DIFF_AVAILABLE }}
-
-            ## CRITICAL DESIGN: Verify First, Then Sync
-
-            You MUST follow this three-phase approach:
-
-            ═══════════════════════════════════════════════════════════════
-            ║  PHASE 1: VERIFY - Analyze and Generate Change Report       ║
-            ═══════════════════════════════════════════════════════════════
-
-            ### Step 1.1: Analyze Git Diff (for incremental mode)
-            Use the Read tool to read `/tmp/i18n-diff.txt` to see the git diff.
-
-            Parse the diff to categorize changes:
-            - Lines with `+` (not `+++`): Added or modified values
-            - Lines with `-` (not `---`): Removed or old values
-            - Identify specific keys for each category:
-              * ADD: Keys that appear only in `+` lines (new keys)
-              * UPDATE: Keys that appear in both `-` and `+` lines (value changed)
-              * DELETE: Keys that appear only in `-` lines (removed keys)
-
-            ### Step 1.2: Read Language Configuration
-            Use the Read tool to read `${{ github.workspace }}/web/i18n-config/languages.ts`.
-            Extract all languages with `supported: true`.
-
-            ### Step 1.3: Run i18n:check for Each Language
-            ```bash
-            pnpm --dir ${{ github.workspace }}/web install --frozen-lockfile
-            ```
-            ```bash
-            pnpm --dir ${{ github.workspace }}/web run i18n:check
-            ```
-
-            This will report:
-            - Missing keys (need to ADD)
-            - Extra keys (need to DELETE)
-
-            ### Step 1.4: Generate Change Report
-
-            Create a structured report identifying:
-            ```
-            ╔══════════════════════════════════════════════════════════════╗
-            ║                    I18N SYNC CHANGE REPORT                   ║
-            ╠══════════════════════════════════════════════════════════════╣
-            ║ Files to process: [list]                                     ║
-            ║ Languages to sync: [list]                                    ║
-            ╠══════════════════════════════════════════════════════════════╣
-            ║ ADD (New Keys):                                              ║
-            ║   - [filename].[key]: "English value"                        ║
-            ║   ...                                                        ║
-            ╠══════════════════════════════════════════════════════════════╣
-            ║ UPDATE (Modified Keys - MUST re-translate):                  ║
-            ║   - [filename].[key]: "Old value" → "New value"              ║
-            ║   ...                                                        ║
-            ╠══════════════════════════════════════════════════════════════╣
-            ║ DELETE (Extra Keys):                                         ║
-            ║   - [language]/[filename].[key]                              ║
-            ║   ...                                                        ║
-            ╚══════════════════════════════════════════════════════════════╝
-            ```
-
-            **IMPORTANT**: For UPDATE detection, compare git diff to find keys where
-            the English value changed. These MUST be re-translated even if target
-            language already has a translation (it's now stale!).
-
-            ═══════════════════════════════════════════════════════════════
-            ║  PHASE 2: SYNC - Execute Changes Based on Report            ║
-            ═══════════════════════════════════════════════════════════════
-
-            ### Step 2.1: Process ADD Operations (BATCH per language file)
-
-            **CRITICAL WORKFLOW for efficiency:**
-            1. First, translate ALL new keys for ALL languages mentally
-            2. Then, for EACH language file, do ONE Edit operation:
-               - Read the file once
-               - Insert ALL new keys at the beginning (right after the opening `{`)
-               - Don't worry about alphabetical order - lint:fix will sort them later
-
-            Example Edit (adding 3 keys to zh-Hans/app.json):
-            ```
-            old_string: '{\n  "accessControl"'
-            new_string: '{\n  "newKey1": "translation1",\n  "newKey2": "translation2",\n  "newKey3": "translation3",\n  "accessControl"'
-            ```
-
-            **IMPORTANT**:
-            - ONE Edit per language file (not one Edit per key!)
-            - Always use the Edit tool. NEVER use bash scripts, node, or jq.
-
-            ### Step 2.2: Process UPDATE Operations
-
-            **IMPORTANT: Special handling for zh-Hans and ja-JP**
-            If zh-Hans or ja-JP files were ALSO modified in the same push:
-            - Run: `git -C ${{ github.workspace }} diff HEAD~1 --name-only` and check for zh-Hans or ja-JP files
-            - If found, it means someone manually translated them. Apply these rules:
-
-            1. **Missing keys**: Still ADD them (completeness required)
-            2. **Existing translations**: Compare with the NEW English value:
-               - If translation is **completely wrong** or **unrelated** → Update it
-               - If translation is **roughly correct** (captures the meaning) → Keep it, respect manual work
-               - When in doubt, **keep the manual translation**
-
-            Example:
-            - English changed: "Save" → "Save Changes"
-            - Manual translation: "保存更改" → Keep it (correct meaning)
-            - Manual translation: "删除" → Update it (completely wrong)
-
-            For other languages:
-            Use Edit tool to replace the old value with the new translation.
-            You can batch multiple updates in one Edit if they are adjacent.
-
-            ### Step 2.3: Process DELETE Operations
-            For extra keys reported by i18n:check:
-            - Run: `pnpm --dir ${{ github.workspace }}/web run i18n:check --auto-remove`
-            - Or manually remove from target language JSON files
-
-            ## Translation Guidelines
-
-            - PRESERVE all placeholders exactly as-is:
-              - `{{variable}}` - Mustache interpolation
-              - `${variable}` - Template literal
-              - `<tag>content</tag>` - HTML tags
-              - `_one`, `_other` - Pluralization suffixes (these are KEY suffixes, not values)
-
-              **CRITICAL: Variable names and tag names MUST stay in English - NEVER translate them**
-
-              ✅ CORRECT examples:
-              - English: "{{count}} items" → Japanese: "{{count}} 個のアイテム"
-              - English: "{{name}} updated" → Korean: "{{name}} 업데이트됨"
-              - English: "<email>{{email}}</email>" → Chinese: "<email>{{email}}</email>"
-              - English: "<CustomLink>Marketplace</CustomLink>" → Japanese: "<CustomLink>マーケットプレイス</CustomLink>"
-
-              ❌ WRONG examples (NEVER do this - will break the application):
-              - "{{count}}" → "{{カウント}}" ❌ (variable name translated to Japanese)
-              - "{{name}}" → "{{이름}}" ❌ (variable name translated to Korean)
-              - "{{email}}" → "{{邮箱}}" ❌ (variable name translated to Chinese)
-              - "<email>" → "<メール>" ❌ (tag name translated)
-              - "<CustomLink>" → "<自定义链接>" ❌ (component name translated)
-
-            - Use appropriate language register (formal/informal) based on existing translations
-            - Match existing translation style in each language
-            - Technical terms: check existing conventions per language
-            - For CJK languages: no spaces between characters unless necessary
-            - For RTL languages (ar-TN, fa-IR): ensure proper text handling
-
-            ## Output Format Requirements
-            - Alphabetical key ordering (if original file uses it)
-            - 2-space indentation
-            - Trailing newline at end of file
-            - Valid JSON (use proper escaping for special characters)
-
-            ═══════════════════════════════════════════════════════════════
-            ║  PHASE 3: RE-VERIFY - Confirm All Issues Resolved           ║
-            ═══════════════════════════════════════════════════════════════
-
-            ### Step 3.1: Run Lint Fix (IMPORTANT!)
-            ```bash
-            pnpm --dir ${{ github.workspace }}/web lint:fix --quiet -- 'i18n/**/*.json'
-            ```
-            This ensures:
-            - JSON keys are sorted alphabetically (jsonc/sort-keys rule)
-            - Valid i18n keys (dify-i18n/valid-i18n-keys rule)
-            - No extra keys (dify-i18n/no-extra-keys rule)
-
-            ### Step 3.2: Run Final i18n Check
-            ```bash
-            pnpm --dir ${{ github.workspace }}/web run i18n:check
-            ```
-
-            ### Step 3.3: Fix Any Remaining Issues
-            If check reports issues:
-            - Go back to PHASE 2 for unresolved items
-            - Repeat until check passes
-
-            ### Step 3.4: Generate Final Summary
-            ```
-            ╔══════════════════════════════════════════════════════════════╗
-            ║                    SYNC COMPLETED SUMMARY                    ║
-            ╠══════════════════════════════════════════════════════════════╣
-            ║ Language      │ Added │ Updated │ Deleted │ Status          ║
-            ╠══════════════════════════════════════════════════════════════╣
-            ║ zh-Hans       │  5    │   2     │    1    │ ✓ Complete      ║
-            ║ ja-JP         │  5    │   2     │    1    │ ✓ Complete      ║
-            ║ ...           │ ...   │  ...    │   ...   │ ...             ║
-            ╠══════════════════════════════════════════════════════════════╣
-            ║ i18n:check    │ PASSED - All keys in sync                   ║
-            ╚══════════════════════════════════════════════════════════════╝
-            ```
-
-            ## Mode-Specific Behavior
-
-            **SYNC_MODE = "incremental"** (default):
-            - Focus on keys identified from git diff
-            - Also check i18n:check output for any missing/extra keys
-            - Efficient for small changes
-
-            **SYNC_MODE = "full"**:
-            - Compare ALL keys between en-US and each language
-            - Run i18n:check to identify all discrepancies
-            - Use for first-time sync or fixing historical issues
-
-            ## Important Notes
-
-            1. Always run i18n:check BEFORE and AFTER making changes
-            2. The check script is the source of truth for missing/extra keys
-            3. For UPDATE scenario: git diff is the source of truth for changed values
-            4. Create a single commit with all translation changes
-            5. If any translation fails, continue with others and report failures
-
-            ═══════════════════════════════════════════════════════════════
-            ║  PHASE 4: COMMIT AND CREATE PR                              ║
-            ═══════════════════════════════════════════════════════════════
-
-            After all translations are complete and verified:
-
-            ### Step 4.1: Check for changes
-            ```bash
-            git -C ${{ github.workspace }} status --porcelain
-            ```
-
-            If there are changes:
-
-            ### Step 4.2: Create a new branch and commit
-            Run these git commands ONE BY ONE (not combined with &&).
-            **IMPORTANT**: Do NOT use `$()` command substitution. Use two separate commands:
-
-            1. First, get the timestamp:
-            ```bash
-            date +%Y%m%d-%H%M%S
-            ```
-            (Note the output, e.g., "20260115-143052")
-
-            2. Then create branch using the timestamp value:
-            ```bash
-            git -C ${{ github.workspace }} checkout -b chore/i18n-sync-20260115-143052
-            ```
-            (Replace "20260115-143052" with the actual timestamp from step 1)
-
-            3. Stage changes:
-            ```bash
-            git -C ${{ github.workspace }} add web/i18n/
-            ```
-
-            4. Commit:
-            ```bash
-            git -C ${{ github.workspace }} commit -m "chore(i18n): sync translations with en-US - Mode: ${{ steps.detect_changes.outputs.SYNC_MODE }}"
-            ```
-
-            5. Push:
-            ```bash
-            git -C ${{ github.workspace }} push origin HEAD
-            ```
-
-            ### Step 4.3: Create Pull Request
-            ```bash
-            gh pr create --repo ${{ github.repository }} --title "chore(i18n): sync translations with en-US" --body "## Summary
-
-            This PR was automatically generated to sync i18n translation files.
-
-            ### Changes
-            - Mode: ${{ steps.detect_changes.outputs.SYNC_MODE }}
-            - Files processed: ${{ steps.detect_changes.outputs.CHANGED_FILES }}
-
-            ### Verification
-            - [x] \`i18n:check\` passed
-            - [x] \`lint:fix\` applied
-
-            🤖 Generated with Claude Code GitHub Action" --base main
-            ```
+            Tool rules:
+            - Use Read for repository files.
+            - Use Edit for JSON updates.
+            - Use Bash only for `git`, `gh`, `pnpm`, and `date`.
+            - Run Bash commands one by one. Do not combine commands with `&&`, `||`, pipes, or command substitution.
 
+            Required execution plan:
+            1. Resolve target languages.
+               - Use the provided `Target languages` value as the source of truth.
+               - If it is unexpectedly empty, read `${{ github.workspace }}/web/i18n-config/languages.ts` and use every language with `supported: true` except `en-US`.
+            2. Stay strictly in scope.
+               - Only process the files listed in `Files in scope`.
+               - Only process the resolved target languages, never `en-US`.
+               - Do not touch unrelated i18n files.
+               - Do not modify `${{ github.workspace }}/web/i18n/en-US/`.
+            3. Detect English changes per file.
+               - Read the current English JSON file for each file in scope.
+               - If sync mode is `incremental` and `Base SHA` is not empty, run:
+                 `git -C ${{ github.workspace }} show <Base SHA>:web/i18n/en-US/<file>.json`
+               - If sync mode is `full` or `Base SHA` is empty, skip historical comparison and treat the current English file as the only source of truth for structural sync.
+               - If the file did not exist at Base SHA, treat all current keys as ADD.
+               - Compare previous and current English JSON to identify:
+                 - ADD: key only in current
+                 - UPDATE: key exists in both and the English value changed
+                 - DELETE: key only in previous
+               - Do not rely on a truncated diff file.
+            4. Run a scoped pre-check before editing:
+               - `pnpm --dir ${{ github.workspace }}/web run i18n:check ${{ steps.context.outputs.FILE_ARGS }} ${{ steps.context.outputs.LANG_ARGS }}`
+               - Use this command as the source of truth for missing and extra keys inside the current scope.
+            5. Apply translations.
+               - For every target language and scoped file:
+                 - If the locale file does not exist yet, create it with `Write` and then continue with `Edit` as needed.
+                 - ADD missing keys.
+                 - UPDATE stale translations when the English value changed.
+                 - DELETE removed keys. Prefer `pnpm --dir ${{ github.workspace }}/web run i18n:check ${{ steps.context.outputs.FILE_ARGS }} ${{ steps.context.outputs.LANG_ARGS }} --auto-remove` for extra keys so deletions stay in scope.
+               - For `zh-Hans` and `ja-JP`, if the locale file also changed between Base SHA and Head SHA, preserve manual translations unless they are clearly wrong for the new English value. If in doubt, keep the manual translation.
+               - Preserve placeholders exactly: `{{variable}}`, `${variable}`, HTML tags, component tags, and variable names.
+               - Match the existing terminology and register used by each locale.
+               - Prefer one Edit per file when stable, but prioritize correctness over batching.
+            6. Verify only the edited files.
+               - Run `pnpm --dir ${{ github.workspace }}/web lint:fix --quiet -- <relative edited i18n file paths>`
+               - Run `pnpm --dir ${{ github.workspace }}/web run i18n:check ${{ steps.context.outputs.FILE_ARGS }} ${{ steps.context.outputs.LANG_ARGS }}`
+               - If verification fails, fix the remaining problems before continuing.
+            7. Create a PR only when there are changes in `web/i18n/`.
+               - Check `git -C ${{ github.workspace }} status --porcelain -- web/i18n/`
+               - Create branch `chore/i18n-sync-<timestamp>`
+               - Commit message: `chore(i18n): sync translations with en-US`
+               - Push the branch and open a PR against `main`
+               - PR title: `chore(i18n): sync translations with en-US`
+               - PR body: summarize files, languages, sync mode, and verification commands
+            8. If there are no translation changes after verification, do not create a branch, commit, or PR.
           claude_args: |
-            --max-turns 150
+            --max-turns 80
             --allowedTools "Read,Write,Edit,Bash(git *),Bash(git:*),Bash(gh *),Bash(gh:*),Bash(pnpm *),Bash(pnpm:*),Bash(date *),Bash(date:*),Glob,Grep"

.github/workflows/trigger-i18n-sync.yml

@@ -1,66 +0,0 @@
-name: Trigger i18n Sync on Push
-
-# This workflow bridges the push event to repository_dispatch
-# because claude-code-action doesn't support push events directly.
-# See: https://github.com/langgenius/dify/issues/30743
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - 'web/i18n/en-US/*.json'
-
-permissions:
-  contents: write
-
-jobs:
-  trigger:
-    if: github.repository == 'langgenius/dify'
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-        with:
-          fetch-depth: 0
-
-      - name: Detect changed files and generate diff
-        id: detect
-        run: |
-          BEFORE_SHA="${{ github.event.before }}"
-          # Handle edge case: force push may have null/zero SHA
-          if [ -z "$BEFORE_SHA" ] || [ "$BEFORE_SHA" = "0000000000000000000000000000000000000000" ]; then
-            BEFORE_SHA="HEAD~1"
-          fi
-
-          # Detect changed i18n files
-          changed=$(git diff --name-only "$BEFORE_SHA" "${{ github.sha }}" -- 'web/i18n/en-US/*.json' 2>/dev/null | xargs -n1 basename 2>/dev/null | sed 's/.json$//' | tr '\n' ' ' || echo "")
-          echo "changed_files=$changed" >> $GITHUB_OUTPUT
-
-          # Generate diff for context
-          git diff "$BEFORE_SHA" "${{ github.sha }}" -- 'web/i18n/en-US/*.json' > /tmp/i18n-diff.txt 2>/dev/null || echo "" > /tmp/i18n-diff.txt
-
-          # Truncate if too large (keep first 50KB to match receiving workflow)
-          head -c 50000 /tmp/i18n-diff.txt > /tmp/i18n-diff-truncated.txt
-          mv /tmp/i18n-diff-truncated.txt /tmp/i18n-diff.txt
-
-          # Base64 encode the diff for safe JSON transport (portable, single-line)
-          diff_base64=$(base64 < /tmp/i18n-diff.txt | tr -d '\n')
-          echo "diff_base64=$diff_base64" >> $GITHUB_OUTPUT
-
-          if [ -n "$changed" ]; then
-            echo "has_changes=true" >> $GITHUB_OUTPUT
-            echo "Detected changed files: $changed"
-          else
-            echo "has_changes=false" >> $GITHUB_OUTPUT
-            echo "No i18n changes detected"
-          fi
-
-      - name: Trigger i18n sync workflow
-        if: steps.detect.outputs.has_changes == 'true'
-        uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          event-type: i18n-sync
-          client-payload: '{"changed_files": "${{ steps.detect.outputs.changed_files }}", "diff_base64": "${{ steps.detect.outputs.diff_base64 }}", "sync_mode": "incremental", "trigger_sha": "${{ github.sha }}"}'

.github/workflows/vdb-tests-full.yml

@@ -0,0 +1,95 @@
+name: Run Full VDB Tests
+
+on:
+  schedule:
+    - cron: '0 3 * * 1'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: vdb-tests-full-${{ github.ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Full VDB Tests
+    if: github.repository == 'langgenius/dify'
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.12"
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: Free Disk Space
+        uses: endersonmenezes/free-disk-space@7901478139cff6e9d44df5972fd8ab8fcade4db1 # v3.2.2
+        with:
+          remove_dotnet: true
+          remove_haskell: true
+          remove_tool_cache: true
+
+      - name: Setup UV and Python
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+          cache-dependency-glob: api/uv.lock
+
+      - name: Check UV lockfile
+        run: uv lock --project api --check
+
+      - name: Install dependencies
+        run: uv sync --project api --dev
+
+      - name: Set up dotenvs
+        run: |
+          cp docker/.env.example docker/.env
+          cp docker/middleware.env.example docker/middleware.env
+
+      - name: Expose Service Ports
+        run: sh .github/workflows/expose_service_ports.sh
+
+#      - name: Set up Vector Store (TiDB)
+#        uses: hoverkraft-tech/compose-action@v2.0.2
+#        with:
+#          compose-file: docker/tidb/docker-compose.yaml
+#          services: |
+#            tidb
+#            tiflash
+
+      - name: Set up Full Vector Store Matrix
+        uses: hoverkraft-tech/compose-action@4894d2492015c1774ee5a13a95b1072093087ec3 # v2.5.0
+        with:
+          compose-file: |
+            docker/docker-compose.yaml
+          services: |
+            weaviate
+            qdrant
+            couchbase-server
+            etcd
+            minio
+            milvus-standalone
+            pgvecto-rs
+            pgvector
+            chroma
+            elasticsearch
+            oceanbase
+
+      - name: setup test config
+        run: |
+          echo $(pwd)
+          ls -lah .
+          cp api/tests/integration_tests/.env.example api/tests/integration_tests/.env
+
+#      - name: Check VDB Ready (TiDB)
+#        run: uv run --project api python api/tests/integration_tests/vdb/tidb_vector/check_tiflash_ready.py
+
+      - name: Test Vector Stores
+        run: uv run --project api bash dev/pytest/pytest_vdb.sh

.github/workflows/vdb-tests.yml

@@ -1,20 +1,22 @@
-name: Run VDB Tests
+name: Run VDB Smoke Tests
 
 on:
   workflow_call:
 
+permissions:
+  contents: read
+
 concurrency:
   group: vdb-tests-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
 jobs:
   test:
-    name: VDB Tests
+    name: VDB Smoke Tests
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version:
-          - "3.11"
           - "3.12"
 
     steps:
@@ -31,7 +33,7 @@ jobs:
           remove_tool_cache: true
 
       - name: Setup UV and Python
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
         with:
           enable-cache: true
           python-version: ${{ matrix.python-version }}
@@ -59,23 +61,18 @@ jobs:
 #            tidb
 #            tiflash
 
-      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch, Couchbase, OceanBase)
+      - name: Set up Vector Stores for Smoke Coverage
         uses: hoverkraft-tech/compose-action@4894d2492015c1774ee5a13a95b1072093087ec3 # v2.5.0
         with:
           compose-file: |
             docker/docker-compose.yaml
           services: |
+            db_postgres
+            redis
             weaviate
             qdrant
-            couchbase-server
-            etcd
-            minio
-            milvus-standalone
-            pgvecto-rs
             pgvector
             chroma
-            elasticsearch
-            oceanbase
 
       - name: setup test config
         run: |
@@ -87,4 +84,9 @@ jobs:
 #        run: uv run --project api python api/tests/integration_tests/vdb/tidb_vector/check_tiflash_ready.py
 
       - name: Test Vector Stores
-        run: uv run --project api bash dev/pytest/pytest_vdb.sh
+        run: |
+          uv run --project api pytest --timeout "${PYTEST_TIMEOUT:-180}" \
+            api/tests/integration_tests/vdb/chroma \
+            api/tests/integration_tests/vdb/pgvector \
+            api/tests/integration_tests/vdb/qdrant \
+            api/tests/integration_tests/vdb/weaviate

.github/workflows/web-e2e.yml

@@ -0,0 +1,68 @@
+name: Web Full-Stack E2E
+
+on:
+  workflow_call:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: web-e2e-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Web Full-Stack E2E
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: Setup web dependencies
+        uses: ./.github/actions/setup-web
+
+      - name: Setup UV and Python
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
+        with:
+          enable-cache: true
+          python-version: "3.12"
+          cache-dependency-glob: api/uv.lock
+
+      - name: Install API dependencies
+        run: uv sync --project api --dev
+
+      - name: Install Playwright browser
+        working-directory: ./e2e
+        run: vp run e2e:install
+
+      - name: Run isolated source-api and built-web Cucumber E2E tests
+        working-directory: ./e2e
+        env:
+          E2E_ADMIN_EMAIL: e2e-admin@example.com
+          E2E_ADMIN_NAME: E2E Admin
+          E2E_ADMIN_PASSWORD: E2eAdmin12345
+          E2E_FORCE_WEB_BUILD: "1"
+          E2E_INIT_PASSWORD: E2eInit12345
+        run: vp run e2e:full
+
+      - name: Upload Cucumber report
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: cucumber-report
+          path: e2e/cucumber-report
+          retention-days: 7
+
+      - name: Upload E2E logs
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: e2e-logs
+          path: e2e/.logs
+          retention-days: 7

.github/workflows/web-tests.yml

@@ -22,8 +22,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        shardIndex: [1, 2, 3, 4, 5, 6]
-        shardTotal: [6]
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
     defaults:
       run:
         shell: bash
@@ -66,7 +66,6 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
-          fetch-depth: 0
           persist-credentials: false
 
       - name: Setup web environment
@@ -84,40 +83,9 @@ jobs:
 
       - name: Report coverage
         if: ${{ env.CODECOV_TOKEN != '' }}
-        uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5.5.3
+        uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
         with:
           directory: web/coverage
           flags: web
         env:
           CODECOV_TOKEN: ${{ env.CODECOV_TOKEN }}
-
-  web-build:
-    name: Web Build
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: ./web
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-        with:
-          persist-credentials: false
-
-      - name: Check changed files
-        id: changed-files
-        uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
-        with:
-          files: |
-            web/**
-            .github/workflows/web-tests.yml
-            .github/actions/setup-web/**
-
-      - name: Setup web environment
-        if: steps.changed-files.outputs.any_changed == 'true'
-        uses: ./.github/actions/setup-web
-
-      - name: Web build check
-        if: steps.changed-files.outputs.any_changed == 'true'
-        working-directory: ./web
-        run: vp run build

.gitignore

@@ -212,6 +212,7 @@ api/.vscode
 
 # pnpm
 /.pnpm-store
+/node_modules
 
 # plugin migrate
 plugins.jsonl
@@ -239,4 +240,4 @@ scripts/stress-test/reports/
 *.local.md
 
 # Code Agent Folder
-.qoder/*
+.qoder/*

Makefile

@@ -24,8 +24,8 @@ prepare-docker:
 # Step 2: Prepare web environment
 prepare-web:
 	@echo "🌐 Setting up web environment..."
-	@cp -n web/.env.example web/.env 2>/dev/null || echo "Web .env already exists"
-	@cd web && pnpm install
+	@cp -n web/.env.example web/.env.local 2>/dev/null || echo "Web .env.local already exists"
+	@pnpm install
 	@echo "✅ Web environment prepared (not started)"
 
 # Step 3: Prepare API environment
@@ -74,6 +74,12 @@ type-check:
 	@uv --directory api run mypy --exclude-gitignore --exclude 'tests/' --exclude 'migrations/' --check-untyped-defs --disable-error-code=import-untyped .
 	@echo "✅ Type checks complete"
 
+type-check-core:
+	@echo "📝 Running core type checks (basedpyright + mypy)..."
+	@./dev/basedpyright-check $(PATH_TO_CHECK)
+	@uv --directory api run mypy --exclude-gitignore --exclude 'tests/' --exclude 'migrations/' --check-untyped-defs --disable-error-code=import-untyped .
+	@echo "✅ Core type checks complete"
+
 test:
 	@echo "🧪 Running backend unit tests..."
 	@if [ -n "$(TARGET_TESTS)" ]; then \
@@ -87,7 +93,7 @@ test:
 # Build Docker images
 build-web:
 	@echo "Building web Docker image: $(WEB_IMAGE):$(VERSION)..."
-	docker build -t $(WEB_IMAGE):$(VERSION) ./web
+	docker build -f web/Dockerfile -t $(WEB_IMAGE):$(VERSION) .
 	@echo "Web Docker image built successfully: $(WEB_IMAGE):$(VERSION)"
 
 build-api:
@@ -133,6 +139,7 @@ help:
 	@echo "  make check          - Check code with ruff"
 	@echo "  make lint           - Format, fix, and lint code (ruff, imports, dotenv)"
 	@echo "  make type-check     - Run type checks (basedpyright, pyrefly, mypy)"
+	@echo "  make type-check-core - Run core type checks (basedpyright, mypy)"
 	@echo "  make test           - Run backend unit tests (or TARGET_TESTS=./api/tests/<target_tests>)"
 	@echo ""
 	@echo "Docker Build Targets:"

README.md

@@ -53,7 +53,11 @@
   <a href="./docs/tr-TR/README.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
   <a href="./docs/vi-VN/README.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
   <a href="./docs/de-DE/README.md"><img alt="README in Deutsch" src="https://img.shields.io/badge/German-d9d9d9"></a>
+  <a href="./docs/it-IT/README.md"><img alt="README in Italiano" src="https://img.shields.io/badge/Italiano-d9d9d9"></a>
+  <a href="./docs/pt-BR/README.md"><img alt="README em Português do Brasil" src="https://img.shields.io/badge/Portugu%C3%AAs%20do%20Brasil-d9d9d9"></a>
+  <a href="./docs/sl-SI/README.md"><img alt="README Slovenščina" src="https://img.shields.io/badge/Sloven%C5%A1%C4%8Dina-d9d9d9"></a>
   <a href="./docs/bn-BD/README.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
+  <a href="./docs/hi-IN/README.md"><img alt="README in हिन्दी" src="https://img.shields.io/badge/Hindi-d9d9d9"></a>
 </p>
 
 Dify is an open-source LLM app development platform. Its intuitive interface combines AI workflow, RAG pipeline, agent capabilities, model management, observability features (including [Opik](https://www.comet.com/docs/opik/integrations/dify), [Langfuse](https://docs.langfuse.com), and [Arize Phoenix](https://docs.arize.com/phoenix)) and more, letting you quickly go from prototype to production. Here's a list of the core features:

api/.env.example

@@ -127,7 +127,8 @@ ALIYUN_OSS_AUTH_VERSION=v1
 ALIYUN_OSS_REGION=your-region
 # Don't start with '/'. OSS doesn't support leading slash in object names.
 ALIYUN_OSS_PATH=your-path
-ALIYUN_CLOUDBOX_ID=your-cloudbox-id
+# Optional CloudBox ID for Aliyun OSS, DO NOT enable it if you are not using CloudBox.
+#ALIYUN_CLOUDBOX_ID=your-cloudbox-id
 
 # Google Storage configuration
 GOOGLE_STORAGE_BUCKET_NAME=your-bucket-name

api/.importlinter

@@ -3,7 +3,6 @@ root_packages =
     core
     constants
     context
-    graphon
     configs
     controllers
     extensions
@@ -13,152 +12,3 @@ root_packages =
     tasks
     services
 include_external_packages = True
-
-[importlinter:contract:workflow]
-name = Workflow
-type=layers
-layers =
-    graph_engine
-    graph_events
-    graph
-    nodes
-    node_events
-    runtime
-    entities
-containers =
-    graphon
-ignore_imports =
-    graphon.nodes.base.node -> graphon.graph_events
-    graphon.nodes.iteration.iteration_node -> graphon.graph_events
-    graphon.nodes.loop.loop_node -> graphon.graph_events
-
-    graphon.nodes.iteration.iteration_node -> graphon.graph_engine
-    graphon.nodes.loop.loop_node -> graphon.graph_engine
-    # TODO(QuantumGhost): fix the import violation later
-    graphon.entities.pause_reason -> graphon.nodes.human_input.entities
-
-[importlinter:contract:workflow-external-imports]
-name = Workflow External Imports
-type = forbidden
-source_modules =
-    graphon
-forbidden_modules =
-    constants
-    configs
-    context
-    controllers
-    extensions
-    factories
-    libs
-    models
-    services
-    tasks
-    core.agent
-    core.app
-    core.base
-    core.callback_handler
-    core.datasource
-    core.db
-    core.entities
-    core.errors
-    core.extension
-    core.external_data_tool
-    core.file
-    core.helper
-    core.hosting_configuration
-    core.indexing_runner
-    core.llm_generator
-    core.logging
-    core.mcp
-    core.memory
-    core.moderation
-    core.ops
-    core.plugin
-    core.prompt
-    core.provider_manager
-    core.rag
-    core.repositories
-    core.schemas
-    core.tools
-    core.trigger
-    core.variables
-
-[importlinter:contract:workflow-third-party-imports]
-name = Workflow Third-Party Imports
-type = forbidden
-source_modules =
-    graphon
-forbidden_modules =
-    sqlalchemy
-
-[importlinter:contract:rsc]
-name = RSC
-type = layers
-layers =
-    graph_engine
-    response_coordinator
-containers =
-    graphon.graph_engine
-
-[importlinter:contract:worker]
-name = Worker
-type = layers
-layers =
-    graph_engine
-    worker
-containers =
-    graphon.graph_engine
-
-[importlinter:contract:graph-engine-architecture]
-name = Graph Engine Architecture
-type = layers
-layers =
-    graph_engine
-    orchestration
-    command_processing
-    event_management
-    error_handler
-    graph_traversal
-    graph_state_manager
-    worker_management
-    domain
-containers =
-    graphon.graph_engine
-
-[importlinter:contract:domain-isolation]
-name = Domain Model Isolation
-type = forbidden
-source_modules =
-    graphon.graph_engine.domain
-forbidden_modules =
-    graphon.graph_engine.worker_management
-    graphon.graph_engine.command_channels
-    graphon.graph_engine.layers
-    graphon.graph_engine.protocols
-
-[importlinter:contract:worker-management]
-name = Worker Management
-type = forbidden
-source_modules =
-    graphon.graph_engine.worker_management
-forbidden_modules =
-    graphon.graph_engine.orchestration
-    graphon.graph_engine.command_processing
-    graphon.graph_engine.event_management
-
-
-[importlinter:contract:graph-traversal-components]
-name = Graph Traversal Components
-type = layers
-layers =
-    edge_processor
-    skip_propagator
-containers =
-    graphon.graph_engine.graph_traversal
-
-[importlinter:contract:command-channels]
-name = Command Channels Independence
-type = independence
-modules =
-    graphon.graph_engine.command_channels.in_memory_channel
-    graphon.graph_engine.command_channels.redis_channel

api/README.md

@@ -40,6 +40,8 @@ The scripts resolve paths relative to their location, so you can run them from a
    ./dev/start-web
    ```
 
+   `./dev/setup` and `./dev/start-web` install JavaScript dependencies through the repository root workspace, so you do not need a separate `cd web && pnpm install` step.
+
 1. Set up your application by visiting `http://localhost:3000`.
 
 1. Start the worker service (async and scheduler tasks, runs from `api`).

api/app_factory.py

@@ -143,6 +143,7 @@ def initialize_extensions(app: DifyApp):
         ext_commands,
         ext_compress,
         ext_database,
+        ext_enterprise_telemetry,
         ext_fastopenapi,
         ext_forward_refs,
         ext_hosting_provider,
@@ -193,6 +194,7 @@ def initialize_extensions(app: DifyApp):
         ext_commands,
         ext_fastopenapi,
         ext_otel,
+        ext_enterprise_telemetry,
         ext_request_logging,
         ext_session_factory,
     ]

api/configs/app_config.py

@@ -8,7 +8,7 @@ from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, Settings
 from libs.file_utils import search_file_upwards
 
 from .deploy import DeploymentConfig
-from .enterprise import EnterpriseFeatureConfig
+from .enterprise import EnterpriseFeatureConfig, EnterpriseTelemetryConfig
 from .extra import ExtraServiceConfig
 from .feature import FeatureConfig
 from .middleware import MiddlewareConfig
@@ -73,6 +73,8 @@ class DifyConfig(
     # Enterprise feature configs
     # **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
     EnterpriseFeatureConfig,
+    # Enterprise telemetry configs
+    EnterpriseTelemetryConfig,
 ):
     model_config = SettingsConfigDict(
         # read from dotenv format config file

api/configs/enterprise/__init__.py

@@ -22,3 +22,52 @@ class EnterpriseFeatureConfig(BaseSettings):
     ENTERPRISE_REQUEST_TIMEOUT: int = Field(
         ge=1, description="Maximum timeout in seconds for enterprise requests", default=5
     )
+
+
+class EnterpriseTelemetryConfig(BaseSettings):
+    """
+    Configuration for enterprise telemetry.
+    """
+
+    ENTERPRISE_TELEMETRY_ENABLED: bool = Field(
+        description="Enable enterprise telemetry collection (also requires ENTERPRISE_ENABLED=true).",
+        default=False,
+    )
+
+    ENTERPRISE_OTLP_ENDPOINT: str = Field(
+        description="Enterprise OTEL collector endpoint.",
+        default="",
+    )
+
+    ENTERPRISE_OTLP_HEADERS: str = Field(
+        description="Auth headers for OTLP export (key=value,key2=value2).",
+        default="",
+    )
+
+    ENTERPRISE_OTLP_PROTOCOL: str = Field(
+        description="OTLP protocol: 'http' or 'grpc' (default: http).",
+        default="http",
+    )
+
+    ENTERPRISE_OTLP_API_KEY: str = Field(
+        description="Bearer token for enterprise OTLP export authentication.",
+        default="",
+    )
+
+    ENTERPRISE_INCLUDE_CONTENT: bool = Field(
+        description="Include input/output content in traces (privacy toggle).",
+        # Setting the default value to False to avoid accidentally log PII data in traces.
+        default=False,
+    )
+
+    ENTERPRISE_SERVICE_NAME: str = Field(
+        description="Service name for OTEL resource.",
+        default="dify",
+    )
+
+    ENTERPRISE_OTEL_SAMPLING_RATE: float = Field(
+        description="Sampling rate for enterprise traces (0.0 to 1.0, default 1.0 = 100%).",
+        default=1.0,
+        ge=0.0,
+        le=1.0,
+    )

api/controllers/common/fields.py

@@ -2,9 +2,9 @@ from __future__ import annotations
 
 from typing import Any, TypeAlias
 
+from graphon.file import helpers as file_helpers
 from pydantic import BaseModel, ConfigDict, computed_field
 
-from graphon.file import helpers as file_helpers
 from models.model import IconType
 
 JSONValue: TypeAlias = str | int | float | bool | None | dict[str, Any] | list[Any]

api/controllers/console/app/app.py

@@ -5,6 +5,8 @@ from typing import Any, Literal, TypeAlias
 
 from flask import request
 from flask_restx import Resource
+from graphon.enums import WorkflowExecutionStatus
+from graphon.file import helpers as file_helpers
 from pydantic import AliasChoices, BaseModel, ConfigDict, Field, computed_field, field_validator
 from sqlalchemy import select
 from sqlalchemy.orm import Session
@@ -27,8 +29,6 @@ from core.ops.ops_trace_manager import OpsTraceManager
 from core.rag.retrieval.retrieval_methods import RetrievalMethod
 from core.trigger.constants import TRIGGER_NODE_TYPES
 from extensions.ext_database import db
-from graphon.enums import WorkflowExecutionStatus
-from graphon.file import helpers as file_helpers
 from libs.login import current_account_with_tenant, login_required
 from models import App, DatasetPermissionEnum, Workflow
 from models.model import IconType

api/controllers/console/app/audio.py

@@ -2,6 +2,7 @@ import logging
 
 from flask import request
 from flask_restx import Resource, fields
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel, Field
 from werkzeug.exceptions import InternalServerError
 
@@ -22,7 +23,6 @@ from controllers.console.app.error import (
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
-from graphon.model_runtime.errors.invoke import InvokeError
 from libs.login import login_required
 from models import App, AppMode
 from services.audio_service import AudioService

api/controllers/console/app/completion.py

@@ -3,6 +3,7 @@ from typing import Any, Literal
 
 from flask import request
 from flask_restx import Resource
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel, Field, field_validator
 from werkzeug.exceptions import InternalServerError, NotFound
 
@@ -26,7 +27,6 @@ from core.errors.error import (
     QuotaExceededError,
 )
 from core.helper.trace_id_helper import get_external_trace_id
-from graphon.model_runtime.errors.invoke import InvokeError
 from libs import helper
 from libs.helper import uuid_value
 from libs.login import current_user, login_required

api/controllers/console/app/generator.py

@@ -1,6 +1,7 @@
 from collections.abc import Sequence
 
 from flask_restx import Resource
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel, Field
 
 from controllers.console import console_ns
@@ -19,7 +20,6 @@ from core.helper.code_executor.python3.python3_code_provider import Python3CodeP
 from core.llm_generator.entities import RuleCodeGeneratePayload, RuleGeneratePayload, RuleStructuredOutputPayload
 from core.llm_generator.llm_generator import LLMGenerator
 from extensions.ext_database import db
-from graphon.model_runtime.errors.invoke import InvokeError
 from libs.login import current_account_with_tenant, login_required
 from models import App
 from services.workflow_service import WorkflowService

api/controllers/console/app/message.py

@@ -3,6 +3,7 @@ from typing import Literal
 
 from flask import request
 from flask_restx import Resource, fields, marshal_with
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel, Field, field_validator
 from sqlalchemy import exists, func, select
 from werkzeug.exceptions import InternalServerError, NotFound
@@ -26,7 +27,6 @@ from core.app.entities.app_invoke_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from extensions.ext_database import db
 from fields.raws import FilesContainedField
-from graphon.model_runtime.errors.invoke import InvokeError
 from libs.helper import TimestampField, uuid_value
 from libs.infinite_scroll_pagination import InfiniteScrollPagination
 from libs.login import current_account_with_tenant, login_required

api/controllers/console/app/workflow.py

@@ -5,6 +5,10 @@ from typing import Any
 
 from flask import abort, request
 from flask_restx import Resource, fields, marshal_with
+from graphon.enums import NodeType
+from graphon.file import File
+from graphon.graph_engine.manager import GraphEngineManager
+from graphon.model_runtime.utils.encoders import jsonable_encoder
 from pydantic import BaseModel, Field, field_validator
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import BadRequest, Forbidden, InternalServerError, NotFound
@@ -35,10 +39,6 @@ from extensions.ext_redis import redis_client
 from factories import file_factory, variable_factory
 from fields.member_fields import simple_account_fields
 from fields.workflow_fields import workflow_fields, workflow_pagination_fields
-from graphon.enums import NodeType
-from graphon.file.models import File
-from graphon.graph_engine.manager import GraphEngineManager
-from graphon.model_runtime.utils.encoders import jsonable_encoder
 from libs import helper
 from libs.datetime_utils import naive_utc_now
 from libs.helper import TimestampField, uuid_value

api/controllers/console/app/workflow_app_log.py

@@ -3,6 +3,7 @@ from datetime import datetime
 from dateutil.parser import isoparse
 from flask import request
 from flask_restx import Resource, marshal_with
+from graphon.enums import WorkflowExecutionStatus
 from pydantic import BaseModel, Field, field_validator
 from sqlalchemy.orm import Session
 
@@ -14,7 +15,6 @@ from fields.workflow_app_log_fields import (
     build_workflow_app_log_pagination_model,
     build_workflow_archived_log_pagination_model,
 )
-from graphon.enums import WorkflowExecutionStatus
 from libs.login import login_required
 from models import App
 from models.model import AppMode

api/controllers/console/app/workflow_draft_variable.py

@@ -5,6 +5,10 @@ from typing import Any, NoReturn, ParamSpec, TypeVar
 
 from flask import Response, request
 from flask_restx import Resource, fields, marshal, marshal_with
+from graphon.file import helpers as file_helpers
+from graphon.variables.segment_group import SegmentGroup
+from graphon.variables.segments import ArrayFileSegment, FileSegment, Segment
+from graphon.variables.types import SegmentType
 from pydantic import BaseModel, Field
 from sqlalchemy.orm import Session
 
@@ -20,10 +24,6 @@ from core.workflow.variable_prefixes import CONVERSATION_VARIABLE_NODE_ID, SYSTE
 from extensions.ext_database import db
 from factories.file_factory import build_from_mapping, build_from_mappings
 from factories.variable_factory import build_segment_with_type
-from graphon.file import helpers as file_helpers
-from graphon.variables.segment_group import SegmentGroup
-from graphon.variables.segments import ArrayFileSegment, FileSegment, Segment
-from graphon.variables.types import SegmentType
 from libs.login import current_user, login_required
 from models import App, AppMode
 from models.workflow import WorkflowDraftVariable

api/controllers/console/app/workflow_run.py

@@ -1,8 +1,10 @@
 from datetime import UTC, datetime, timedelta
-from typing import Literal, cast
+from typing import Literal, TypedDict, cast
 
 from flask import request
 from flask_restx import Resource, fields, marshal_with
+from graphon.entities.pause_reason import HumanInputRequired
+from graphon.enums import WorkflowExecutionStatus
 from pydantic import BaseModel, Field, field_validator
 from sqlalchemy import select
 from sqlalchemy.orm import sessionmaker
@@ -26,8 +28,6 @@ from fields.workflow_run_fields import (
     workflow_run_node_execution_list_fields,
     workflow_run_pagination_fields,
 )
-from graphon.entities.pause_reason import HumanInputRequired
-from graphon.enums import WorkflowExecutionStatus
 from libs.archive_storage import ArchiveStorageNotConfiguredError, get_archive_storage
 from libs.custom_inputs import time_duration
 from libs.helper import uuid_value
@@ -173,6 +173,23 @@ console_ns.schema_model(
 )
 
 
+class HumanInputPauseTypeResponse(TypedDict):
+    type: Literal["human_input"]
+    form_id: str
+    backstage_input_url: str | None
+
+
+class PausedNodeResponse(TypedDict):
+    node_id: str
+    node_title: str
+    pause_type: HumanInputPauseTypeResponse
+
+
+class WorkflowPauseDetailsResponse(TypedDict):
+    paused_at: str | None
+    paused_nodes: list[PausedNodeResponse]
+
+
 @console_ns.route("/apps/<uuid:app_id>/advanced-chat/workflow-runs")
 class AdvancedChatAppWorkflowRunListApi(Resource):
     @console_ns.doc("get_advanced_chat_workflow_runs")
@@ -490,10 +507,11 @@ class ConsoleWorkflowPauseDetailsApi(Resource):
         # Check if workflow is suspended
         is_paused = workflow_run.status == WorkflowExecutionStatus.PAUSED
         if not is_paused:
-            return {
+            empty_response: WorkflowPauseDetailsResponse = {
                 "paused_at": None,
                 "paused_nodes": [],
-            }, 200
+            }
+            return empty_response, 200
 
         pause_entity = workflow_run_repo.get_workflow_pause(workflow_run_id)
         pause_reasons = pause_entity.get_pause_reasons() if pause_entity else []
@@ -503,8 +521,8 @@ class ConsoleWorkflowPauseDetailsApi(Resource):
 
         # Build response
         paused_at = pause_entity.paused_at if pause_entity else None
-        paused_nodes = []
-        response = {
+        paused_nodes: list[PausedNodeResponse] = []
+        response: WorkflowPauseDetailsResponse = {
             "paused_at": paused_at.isoformat() + "Z" if paused_at else None,
             "paused_nodes": paused_nodes,
         }

api/controllers/console/auth/oauth_server.py

@@ -4,11 +4,11 @@ from typing import Concatenate, ParamSpec, TypeVar
 
 from flask import jsonify, request
 from flask_restx import Resource
+from graphon.model_runtime.utils.encoders import jsonable_encoder
 from pydantic import BaseModel
 from werkzeug.exceptions import BadRequest, NotFound
 
 from controllers.console.wraps import account_initialization_required, setup_required
-from graphon.model_runtime.utils.encoders import jsonable_encoder
 from libs.login import current_account_with_tenant, login_required
 from models import Account
 from models.model import OAuthProviderApp

api/controllers/console/datasets/datasets.py

@@ -2,6 +2,7 @@ from typing import Any, cast
 
 from flask import request
 from flask_restx import Resource, fields, marshal, marshal_with
+from graphon.model_runtime.entities.model_entities import ModelType
 from pydantic import BaseModel, Field, field_validator
 from sqlalchemy import func, select
 from werkzeug.exceptions import Forbidden, NotFound
@@ -51,7 +52,6 @@ from fields.dataset_fields import (
     weighted_score_fields,
 )
 from fields.document_fields import document_status_fields
-from graphon.model_runtime.entities.model_entities import ModelType
 from libs.login import current_account_with_tenant, login_required
 from models import ApiToken, Dataset, Document, DocumentSegment, UploadFile
 from models.dataset import DatasetPermission, DatasetPermissionEnum

api/controllers/console/datasets/datasets_document.py

@@ -9,6 +9,8 @@ from uuid import UUID
 import sqlalchemy as sa
 from flask import request, send_file
 from flask_restx import Resource, fields, marshal, marshal_with
+from graphon.model_runtime.entities.model_entities import ModelType
+from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
 from pydantic import BaseModel, Field
 from sqlalchemy import asc, desc, func, select
 from werkzeug.exceptions import Forbidden, NotFound
@@ -37,8 +39,6 @@ from fields.document_fields import (
     document_status_fields,
     document_with_segments_fields,
 )
-from graphon.model_runtime.entities.model_entities import ModelType
-from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
 from libs.datetime_utils import naive_utc_now
 from libs.login import current_account_with_tenant, login_required
 from models import DatasetProcessRule, Document, DocumentSegment, UploadFile

api/controllers/console/datasets/datasets_segments.py

@@ -2,6 +2,7 @@ import uuid
 
 from flask import request
 from flask_restx import Resource, marshal
+from graphon.model_runtime.entities.model_entities import ModelType
 from pydantic import BaseModel, Field
 from sqlalchemy import String, cast, func, or_, select
 from sqlalchemy.dialects.postgresql import JSONB
@@ -30,7 +31,6 @@ from core.rag.index_processor.constant.index_type import IndexTechniqueType
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from fields.segment_fields import child_chunk_fields, segment_fields
-from graphon.model_runtime.entities.model_entities import ModelType
 from libs.helper import escape_like_pattern
 from libs.login import current_account_with_tenant, login_required
 from models.dataset import ChildChunk, DocumentSegment

api/controllers/console/datasets/hit_testing_base.py

@@ -2,6 +2,7 @@ import logging
 from typing import Any
 
 from flask_restx import marshal
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel, Field
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
@@ -20,7 +21,6 @@ from core.errors.error import (
     QuotaExceededError,
 )
 from fields.hit_testing_fields import hit_testing_record_fields
-from graphon.model_runtime.errors.invoke import InvokeError
 from libs.login import current_user
 from models.account import Account
 from services.dataset_service import DatasetService

api/controllers/console/datasets/rag_pipeline/datasource_auth.py

@@ -2,6 +2,8 @@ from typing import Any
 
 from flask import make_response, redirect, request
 from flask_restx import Resource
+from graphon.model_runtime.errors.validate import CredentialsValidateFailedError
+from graphon.model_runtime.utils.encoders import jsonable_encoder
 from pydantic import BaseModel, Field
 from werkzeug.exceptions import Forbidden, NotFound
 
@@ -10,8 +12,6 @@ from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.wraps import account_initialization_required, edit_permission_required, setup_required
 from core.plugin.impl.oauth import OAuthHandler
-from graphon.model_runtime.errors.validate import CredentialsValidateFailedError
-from graphon.model_runtime.utils.encoders import jsonable_encoder
 from libs.login import current_account_with_tenant, login_required
 from models.provider_ids import DatasourceProviderID
 from services.datasource_provider_service import DatasourceProviderService
@@ -120,7 +120,8 @@ class DatasourceOAuthCallback(Resource):
         if context is None:
             raise Forbidden("Invalid context_id")
 
-        user_id, tenant_id = context.get("user_id"), context.get("tenant_id")
+        user_id: str = context["user_id"]
+        tenant_id: str = context["tenant_id"]
         datasource_provider_id = DatasourceProviderID(provider_id)
         plugin_id = datasource_provider_id.plugin_id
         datasource_provider_service = DatasourceProviderService()
@@ -141,7 +142,7 @@ class DatasourceOAuthCallback(Resource):
             system_credentials=oauth_client_params,
             request=request,
         )
-        credential_id = context.get("credential_id")
+        credential_id: str | None = context.get("credential_id")
         if credential_id:
             datasource_provider_service.reauthorize_datasource_oauth_provider(
                 tenant_id=tenant_id,
@@ -150,7 +151,7 @@ class DatasourceOAuthCallback(Resource):
                 name=oauth_response.metadata.get("name") or None,
                 expire_at=oauth_response.expires_at,
                 credentials=dict(oauth_response.credentials),
-                credential_id=context.get("credential_id"),
+                credential_id=credential_id,
             )
         else:
             datasource_provider_service.add_datasource_oauth_provider(

api/controllers/console/datasets/rag_pipeline/rag_pipeline_draft_variable.py

@@ -3,6 +3,7 @@ from typing import Any, NoReturn
 
 from flask import Response, request
 from flask_restx import Resource, marshal, marshal_with
+from graphon.variables.types import SegmentType
 from pydantic import BaseModel, Field
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden
@@ -26,7 +27,6 @@ from core.workflow.variable_prefixes import CONVERSATION_VARIABLE_NODE_ID, SYSTE
 from extensions.ext_database import db
 from factories.file_factory import build_from_mapping, build_from_mappings
 from factories.variable_factory import build_segment_with_type
-from graphon.variables.types import SegmentType
 from libs.login import current_user, login_required
 from models import Account
 from models.dataset import Pipeline

api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py

@@ -4,6 +4,7 @@ from typing import Any, Literal, cast
 
 from flask import abort, request
 from flask_restx import Resource, marshal_with  # type: ignore
+from graphon.model_runtime.utils.encoders import jsonable_encoder
 from pydantic import BaseModel, Field
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import BadRequest, Forbidden, InternalServerError, NotFound
@@ -39,7 +40,6 @@ from core.app.apps.pipeline.pipeline_generator import PipelineGenerator
 from core.app.entities.app_invoke_entities import InvokeFrom
 from extensions.ext_database import db
 from factories import variable_factory
-from graphon.model_runtime.utils.encoders import jsonable_encoder
 from libs import helper
 from libs.helper import TimestampField, UUIDStrOrEmpty
 from libs.login import current_account_with_tenant, current_user, login_required
@@ -53,6 +53,7 @@ from services.rag_pipeline.pipeline_generate_service import PipelineGenerateServ
 from services.rag_pipeline.rag_pipeline import RagPipelineService
 from services.rag_pipeline.rag_pipeline_manage_service import RagPipelineManageService
 from services.rag_pipeline.rag_pipeline_transform_service import RagPipelineTransformService
+from services.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError, WorkflowService
 
 logger = logging.getLogger(__name__)
 
@@ -781,7 +782,38 @@ class RagPipelineByIdApi(Resource):
             # Commit the transaction in the controller
             session.commit()
 
-        return workflow
+            return workflow
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    @get_rag_pipeline
+    def delete(self, pipeline: Pipeline, workflow_id: str):
+        """
+        Delete a published workflow version that is not currently active on the pipeline.
+        """
+        if pipeline.workflow_id == workflow_id:
+            abort(400, description=f"Cannot delete workflow that is currently in use by pipeline '{pipeline.id}'")
+
+        workflow_service = WorkflowService()
+
+        with Session(db.engine) as session:
+            try:
+                workflow_service.delete_workflow(
+                    session=session,
+                    workflow_id=workflow_id,
+                    tenant_id=pipeline.tenant_id,
+                )
+                session.commit()
+            except WorkflowInUseError as e:
+                abort(400, description=str(e))
+            except DraftWorkflowDeletionError as e:
+                abort(400, description=str(e))
+            except ValueError as e:
+                raise NotFound(str(e))
+
+        return None, 204
 
 
 @console_ns.route("/rag/pipelines/<uuid:pipeline_id>/workflows/published/processing/parameters")

api/controllers/console/explore/audio.py

@@ -1,6 +1,7 @@
 import logging
 
 from flask import request
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel, Field
 from werkzeug.exceptions import InternalServerError
 
@@ -19,7 +20,6 @@ from controllers.console.app.error import (
 )
 from controllers.console.explore.wraps import InstalledAppResource
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
-from graphon.model_runtime.errors.invoke import InvokeError
 from services.audio_service import AudioService
 from services.errors.audio import (
     AudioTooLargeServiceError,

api/controllers/console/explore/completion.py

@@ -2,6 +2,7 @@ import logging
 from typing import Any, Literal
 from uuid import UUID
 
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel, Field, field_validator
 from werkzeug.exceptions import InternalServerError, NotFound
 
@@ -25,7 +26,6 @@ from core.errors.error import (
     QuotaExceededError,
 )
 from extensions.ext_database import db
-from graphon.model_runtime.errors.invoke import InvokeError
 from libs import helper
 from libs.datetime_utils import naive_utc_now
 from libs.login import current_user

api/controllers/console/explore/message.py

@@ -2,6 +2,7 @@ import logging
 from typing import Literal
 
 from flask import request
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel, Field, TypeAdapter
 from werkzeug.exceptions import InternalServerError, NotFound
 
@@ -23,7 +24,6 @@ from core.app.entities.app_invoke_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from fields.conversation_fields import ResultResponse
 from fields.message_fields import MessageInfiniteScrollPagination, MessageListItem, SuggestedQuestionsResponse
-from graphon.model_runtime.errors.invoke import InvokeError
 from libs import helper
 from libs.helper import UUIDStrOrEmpty
 from libs.login import current_account_with_tenant

api/controllers/console/explore/trial.py

@@ -3,6 +3,8 @@ from typing import Any, Literal, cast
 
 from flask import request
 from flask_restx import Resource, fields, marshal, marshal_with
+from graphon.graph_engine.manager import GraphEngineManager
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel
 from sqlalchemy import select
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
@@ -59,8 +61,6 @@ from fields.workflow_fields import (
     workflow_fields,
     workflow_partial_fields,
 )
-from graphon.graph_engine.manager import GraphEngineManager
-from graphon.model_runtime.errors.invoke import InvokeError
 from libs import helper
 from libs.helper import uuid_value
 from libs.login import current_user

api/controllers/console/explore/workflow.py

@@ -1,6 +1,8 @@
 import logging
 from typing import Any
 
+from graphon.graph_engine.manager import GraphEngineManager
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel
 from werkzeug.exceptions import InternalServerError
 
@@ -22,8 +24,6 @@ from core.errors.error import (
     QuotaExceededError,
 )
 from extensions.ext_redis import redis_client
-from graphon.graph_engine.manager import GraphEngineManager
-from graphon.model_runtime.errors.invoke import InvokeError
 from libs import helper
 from libs.login import current_account_with_tenant
 from models.model import AppMode, InstalledApp

api/controllers/console/human_input_form.py

@@ -15,6 +15,7 @@ from controllers.console import console_ns
 from controllers.console.wraps import account_initialization_required, setup_required
 from controllers.web.error import InvalidArgumentError, NotFoundError
 from core.app.apps.advanced_chat.app_generator import AdvancedChatAppGenerator
+from core.app.apps.base_app_generator import BaseAppGenerator
 from core.app.apps.common.workflow_response_converter import WorkflowResponseConverter
 from core.app.apps.message_generator import MessageGenerator
 from core.app.apps.workflow.app_generator import WorkflowAppGenerator
@@ -166,6 +167,7 @@ class ConsoleWorkflowEventsApi(Resource):
 
         else:
             msg_generator = MessageGenerator()
+            generator: BaseAppGenerator
             if app.mode == AppMode.ADVANCED_CHAT:
                 generator = AdvancedChatAppGenerator()
             elif app.mode == AppMode.WORKFLOW:
@@ -202,7 +204,7 @@ class ConsoleWorkflowEventsApi(Resource):
         )
 
 
-def _retrieve_app_for_workflow_run(session: Session, workflow_run: WorkflowRun):
+def _retrieve_app_for_workflow_run(session: Session, workflow_run: WorkflowRun) -> App:
     query = select(App).where(
         App.id == workflow_run.app_id,
         App.tenant_id == workflow_run.tenant_id,

api/controllers/console/remote_files.py

@@ -2,6 +2,7 @@ import urllib.parse
 
 import httpx
 from flask_restx import Resource
+from graphon.file import helpers as file_helpers
 from pydantic import BaseModel, Field
 
 import services
@@ -15,7 +16,6 @@ from controllers.console import console_ns
 from core.helper import ssrf_proxy
 from extensions.ext_database import db
 from fields.file_fields import FileWithSignedUrl, RemoteFileInfo
-from graphon.file import helpers as file_helpers
 from libs.login import current_account_with_tenant, login_required
 from services.file_service import FileService
 

api/controllers/console/workspace/agent_providers.py

@@ -1,8 +1,8 @@
 from flask_restx import Resource, fields
+from graphon.model_runtime.utils.encoders import jsonable_encoder
 
 from controllers.console import console_ns
 from controllers.console.wraps import account_initialization_required, setup_required
-from graphon.model_runtime.utils.encoders import jsonable_encoder
 from libs.login import current_account_with_tenant, login_required
 from services.agent_service import AgentService
 

api/controllers/console/workspace/endpoint.py

@@ -2,13 +2,13 @@ from typing import Any
 
 from flask import request
 from flask_restx import Resource
+from graphon.model_runtime.utils.encoders import jsonable_encoder
 from pydantic import BaseModel, Field
 
 from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.wraps import account_initialization_required, is_admin_or_owner_required, setup_required
 from core.plugin.impl.exc import PluginPermissionDeniedError
-from graphon.model_runtime.utils.encoders import jsonable_encoder
 from libs.login import current_account_with_tenant, login_required
 from services.plugin.endpoint_service import EndpointService
 

api/controllers/console/workspace/load_balancing_config.py

@@ -1,12 +1,12 @@
 from flask_restx import Resource
+from graphon.model_runtime.entities.model_entities import ModelType
+from graphon.model_runtime.errors.validate import CredentialsValidateFailedError
 from pydantic import BaseModel
 from werkzeug.exceptions import Forbidden
 
 from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.wraps import account_initialization_required, setup_required
-from graphon.model_runtime.entities.model_entities import ModelType
-from graphon.model_runtime.errors.validate import CredentialsValidateFailedError
 from libs.login import current_account_with_tenant, login_required
 from models import TenantAccountRole
 from services.model_load_balancing_service import ModelLoadBalancingService

api/controllers/console/workspace/model_providers.py

@@ -3,13 +3,13 @@ from typing import Any, Literal
 
 from flask import request, send_file
 from flask_restx import Resource
+from graphon.model_runtime.entities.model_entities import ModelType
+from graphon.model_runtime.errors.validate import CredentialsValidateFailedError
+from graphon.model_runtime.utils.encoders import jsonable_encoder
 from pydantic import BaseModel, Field, field_validator
 
 from controllers.console import console_ns
 from controllers.console.wraps import account_initialization_required, is_admin_or_owner_required, setup_required
-from graphon.model_runtime.entities.model_entities import ModelType
-from graphon.model_runtime.errors.validate import CredentialsValidateFailedError
-from graphon.model_runtime.utils.encoders import jsonable_encoder
 from libs.helper import uuid_value
 from libs.login import current_account_with_tenant, login_required
 from services.billing_service import BillingService

api/controllers/console/workspace/models.py

@@ -3,14 +3,14 @@ from typing import Any, cast
 
 from flask import request
 from flask_restx import Resource
+from graphon.model_runtime.entities.model_entities import ModelType
+from graphon.model_runtime.errors.validate import CredentialsValidateFailedError
+from graphon.model_runtime.utils.encoders import jsonable_encoder
 from pydantic import BaseModel, Field, field_validator
 
 from controllers.common.schema import register_enum_models, register_schema_models
 from controllers.console import console_ns
 from controllers.console.wraps import account_initialization_required, is_admin_or_owner_required, setup_required
-from graphon.model_runtime.entities.model_entities import ModelType
-from graphon.model_runtime.errors.validate import CredentialsValidateFailedError
-from graphon.model_runtime.utils.encoders import jsonable_encoder
 from libs.helper import uuid_value
 from libs.login import current_account_with_tenant, login_required
 from services.model_load_balancing_service import ModelLoadBalancingService
@@ -287,12 +287,10 @@ class ModelProviderModelCredentialApi(Resource):
                 provider=provider,
             )
         else:
-            # Normalize model_type to the origin value stored in DB (e.g., "text-generation" for LLM)
-            normalized_model_type = args.model_type.to_origin_model_type()
             available_credentials = model_provider_service.get_provider_model_available_credentials(
                 tenant_id=tenant_id,
                 provider=provider,
-                model_type=normalized_model_type,
+                model_type=args.model_type,
                 model=args.model,
             )
 

api/controllers/console/workspace/plugin.py

@@ -4,6 +4,7 @@ from typing import Any, Literal
 
 from flask import request, send_file
 from flask_restx import Resource
+from graphon.model_runtime.utils.encoders import jsonable_encoder
 from pydantic import BaseModel, Field
 from werkzeug.datastructures import FileStorage
 from werkzeug.exceptions import Forbidden
@@ -14,7 +15,6 @@ from controllers.console import console_ns
 from controllers.console.workspace import plugin_permission_required
 from controllers.console.wraps import account_initialization_required, is_admin_or_owner_required, setup_required
 from core.plugin.impl.exc import PluginDaemonClientSideError
-from graphon.model_runtime.utils.encoders import jsonable_encoder
 from libs.login import current_account_with_tenant, login_required
 from models.account import TenantPluginAutoUpgradeStrategy, TenantPluginPermission
 from services.plugin.plugin_auto_upgrade_service import PluginAutoUpgradeService
@@ -200,7 +200,7 @@ class PluginDebuggingKeyApi(Resource):
                 "port": dify_config.PLUGIN_REMOTE_INSTALL_PORT,
             }
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
 
 @console_ns.route("/workspaces/current/plugin/list")
@@ -215,7 +215,7 @@ class PluginListApi(Resource):
         try:
             plugins_with_total = PluginService.list_with_total(tenant_id, args.page, args.page_size)
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
         return jsonable_encoder({"plugins": plugins_with_total.list, "total": plugins_with_total.total})
 
@@ -232,7 +232,7 @@ class PluginListLatestVersionsApi(Resource):
         try:
             versions = PluginService.list_latest_versions(args.plugin_ids)
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
         return jsonable_encoder({"versions": versions})
 
@@ -251,7 +251,7 @@ class PluginListInstallationsFromIdsApi(Resource):
         try:
             plugins = PluginService.list_installations_from_ids(tenant_id, args.plugin_ids)
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
         return jsonable_encoder({"plugins": plugins})
 
@@ -266,7 +266,7 @@ class PluginIconApi(Resource):
         try:
             icon_bytes, mimetype = PluginService.get_asset(args.tenant_id, args.filename)
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
         icon_cache_max_age = dify_config.TOOL_ICON_CACHE_MAX_AGE
         return send_file(io.BytesIO(icon_bytes), mimetype=mimetype, max_age=icon_cache_max_age)
@@ -286,7 +286,7 @@ class PluginAssetApi(Resource):
             binary = PluginService.extract_asset(tenant_id, args.plugin_unique_identifier, args.file_name)
             return send_file(io.BytesIO(binary), mimetype="application/octet-stream")
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
 
 @console_ns.route("/workspaces/current/plugin/upload/pkg")
@@ -303,7 +303,7 @@ class PluginUploadFromPkgApi(Resource):
         try:
             response = PluginService.upload_pkg(tenant_id, content)
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
         return jsonable_encoder(response)
 
@@ -323,7 +323,7 @@ class PluginUploadFromGithubApi(Resource):
         try:
             response = PluginService.upload_pkg_from_github(tenant_id, args.repo, args.version, args.package)
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
         return jsonable_encoder(response)
 
@@ -361,7 +361,7 @@ class PluginInstallFromPkgApi(Resource):
         try:
             response = PluginService.install_from_local_pkg(tenant_id, args.plugin_unique_identifiers)
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
         return jsonable_encoder(response)
 
@@ -387,7 +387,7 @@ class PluginInstallFromGithubApi(Resource):
                 args.package,
             )
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
         return jsonable_encoder(response)
 
@@ -407,7 +407,7 @@ class PluginInstallFromMarketplaceApi(Resource):
         try:
             response = PluginService.install_from_marketplace_pkg(tenant_id, args.plugin_unique_identifiers)
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
         return jsonable_encoder(response)
 
@@ -433,7 +433,7 @@ class PluginFetchMarketplacePkgApi(Resource):
                 }
             )
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
 
 @console_ns.route("/workspaces/current/plugin/fetch-manifest")
@@ -453,7 +453,7 @@ class PluginFetchManifestApi(Resource):
                 {"manifest": PluginService.fetch_plugin_manifest(tenant_id, args.plugin_unique_identifier).model_dump()}
             )
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
 
 @console_ns.route("/workspaces/current/plugin/tasks")
@@ -471,7 +471,7 @@ class PluginFetchInstallTasksApi(Resource):
         try:
             return jsonable_encoder({"tasks": PluginService.fetch_install_tasks(tenant_id, args.page, args.page_size)})
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
 
 @console_ns.route("/workspaces/current/plugin/tasks/<task_id>")
@@ -486,7 +486,7 @@ class PluginFetchInstallTaskApi(Resource):
         try:
             return jsonable_encoder({"task": PluginService.fetch_install_task(tenant_id, task_id)})
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
 
 @console_ns.route("/workspaces/current/plugin/tasks/<task_id>/delete")
@@ -501,7 +501,7 @@ class PluginDeleteInstallTaskApi(Resource):
         try:
             return {"success": PluginService.delete_install_task(tenant_id, task_id)}
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
 
 @console_ns.route("/workspaces/current/plugin/tasks/delete_all")
@@ -516,7 +516,7 @@ class PluginDeleteAllInstallTaskItemsApi(Resource):
         try:
             return {"success": PluginService.delete_all_install_task_items(tenant_id)}
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
 
 @console_ns.route("/workspaces/current/plugin/tasks/<task_id>/delete/<path:identifier>")
@@ -531,7 +531,7 @@ class PluginDeleteInstallTaskItemApi(Resource):
         try:
             return {"success": PluginService.delete_install_task_item(tenant_id, task_id, identifier)}
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
 
 @console_ns.route("/workspaces/current/plugin/upgrade/marketplace")
@@ -553,7 +553,7 @@ class PluginUpgradeFromMarketplaceApi(Resource):
                 )
             )
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
 
 @console_ns.route("/workspaces/current/plugin/upgrade/github")
@@ -580,7 +580,7 @@ class PluginUpgradeFromGithubApi(Resource):
                 )
             )
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
 
 @console_ns.route("/workspaces/current/plugin/uninstall")
@@ -598,7 +598,7 @@ class PluginUninstallApi(Resource):
         try:
             return {"success": PluginService.uninstall(tenant_id, args.plugin_installation_id)}
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
 
 @console_ns.route("/workspaces/current/plugin/permission/change")
@@ -674,7 +674,7 @@ class PluginFetchDynamicSelectOptionsApi(Resource):
                 provider_type=args.provider_type,
             )
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
         return jsonable_encoder({"options": options})
 
@@ -705,7 +705,7 @@ class PluginFetchDynamicSelectOptionsWithCredentialsApi(Resource):
                 credentials=args.credentials,
             )
         except PluginDaemonClientSideError as e:
-            raise ValueError(e)
+            return {"code": "plugin_error", "message": e.description}, 400
 
         return jsonable_encoder({"options": options})
 

api/controllers/console/workspace/tool_providers.py

@@ -5,6 +5,7 @@ from urllib.parse import urlparse
 
 from flask import make_response, redirect, request, send_file
 from flask_restx import Resource
+from graphon.model_runtime.utils.encoders import jsonable_encoder
 from pydantic import BaseModel, Field, HttpUrl, field_validator, model_validator
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden
@@ -27,7 +28,6 @@ from core.plugin.entities.plugin_daemon import CredentialType
 from core.plugin.impl.oauth import OAuthHandler
 from core.tools.entities.tool_entities import ApiProviderSchemaType, WorkflowToolParameterConfiguration
 from extensions.ext_database import db
-from graphon.model_runtime.utils.encoders import jsonable_encoder
 from libs.helper import alphanumeric, uuid_value
 from libs.login import current_account_with_tenant, login_required
 from models.provider_ids import ToolProviderID
@@ -832,7 +832,8 @@ class ToolOAuthCallback(Resource):
         tool_provider = ToolProviderID(provider)
         plugin_id = tool_provider.plugin_id
         provider_name = tool_provider.provider_name
-        user_id, tenant_id = context.get("user_id"), context.get("tenant_id")
+        user_id: str = context["user_id"]
+        tenant_id: str = context["tenant_id"]
 
         oauth_handler = OAuthHandler()
         oauth_client_params = BuiltinToolManageService.get_oauth_client(tenant_id, provider)

api/controllers/console/workspace/trigger_providers.py

@@ -3,6 +3,7 @@ from typing import Any
 
 from flask import make_response, redirect, request
 from flask_restx import Resource
+from graphon.model_runtime.utils.encoders import jsonable_encoder
 from pydantic import BaseModel, model_validator
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import BadRequest, Forbidden
@@ -15,7 +16,6 @@ from core.plugin.impl.oauth import OAuthHandler
 from core.trigger.entities.entities import SubscriptionBuilderUpdater
 from core.trigger.trigger_manager import TriggerManager
 from extensions.ext_database import db
-from graphon.model_runtime.utils.encoders import jsonable_encoder
 from libs.login import current_user, login_required
 from models.account import Account
 from models.provider_ids import TriggerProviderID
@@ -499,9 +499,9 @@ class TriggerOAuthCallbackApi(Resource):
         provider_id = TriggerProviderID(provider)
         plugin_id = provider_id.plugin_id
         provider_name = provider_id.provider_name
-        user_id = context.get("user_id")
-        tenant_id = context.get("tenant_id")
-        subscription_builder_id = context.get("subscription_builder_id")
+        user_id: str = context["user_id"]
+        tenant_id: str = context["tenant_id"]
+        subscription_builder_id: str = context["subscription_builder_id"]
 
         # Get OAuth client configuration
         oauth_client_params = TriggerProviderService.get_oauth_client(

api/controllers/inner_api/app/dsl.py

@@ -8,6 +8,7 @@ Go admin-api caller.
 from flask import request
 from flask_restx import Resource
 from pydantic import BaseModel, Field
+from sqlalchemy import select
 from sqlalchemy.orm import Session
 
 from controllers.common.schema import register_schema_model
@@ -87,7 +88,7 @@ class EnterpriseAppDSLExport(Resource):
         """Export an app's DSL as YAML."""
         include_secret = request.args.get("include_secret", "false").lower() == "true"
 
-        app_model = db.session.query(App).filter_by(id=app_id).first()
+        app_model = db.session.get(App, app_id)
         if not app_model:
             return {"message": "app not found"}, 404
 
@@ -104,7 +105,7 @@ def _get_active_account(email: str) -> Account | None:
 
     Workspace membership is already validated by the Go admin-api caller.
     """
-    account = db.session.query(Account).filter_by(email=email).first()
+    account = db.session.scalar(select(Account).where(Account.email == email).limit(1))
     if account is None or account.status != AccountStatus.ACTIVE:
         return None
     return account

api/controllers/inner_api/plugin/plugin.py

@@ -1,4 +1,5 @@
 from flask_restx import Resource
+from graphon.model_runtime.utils.encoders import jsonable_encoder
 
 from controllers.console.wraps import setup_required
 from controllers.inner_api import inner_api_ns
@@ -29,7 +30,6 @@ from core.plugin.entities.request import (
 )
 from core.tools.entities.tool_entities import ToolProviderType
 from core.tools.signature import get_signed_file_url_for_plugin
-from graphon.model_runtime.utils.encoders import jsonable_encoder
 from libs.helper import length_prefixed_response
 from models import Account, Tenant
 from models.model import EndUser

api/controllers/mcp/mcp.py

@@ -2,6 +2,7 @@ from typing import Any, Union
 
 from flask import Response
 from flask_restx import Resource
+from graphon.variables.input_entities import VariableEntity
 from pydantic import BaseModel, Field, ValidationError
 from sqlalchemy.orm import Session
 
@@ -10,7 +11,6 @@ from controllers.mcp import mcp_ns
 from core.mcp import types as mcp_types
 from core.mcp.server.streamable_http import handle_mcp_request
 from extensions.ext_database import db
-from graphon.variables.input_entities import VariableEntity
 from libs import helper
 from models.enums import AppMCPServerStatus
 from models.model import App, AppMCPServer, AppMode, EndUser
@@ -174,6 +174,7 @@ class MCPAppApi(Resource):
             required=variable.get("required", False),
             max_length=variable.get("max_length"),
             options=variable.get("options") or [],
+            json_schema=variable.get("json_schema"),
         )
 
     def _parse_mcp_request(self, args: dict) -> mcp_types.ClientRequest | mcp_types.ClientNotification:

api/controllers/service_api/app/audio.py

@@ -2,6 +2,7 @@ import logging
 
 from flask import request
 from flask_restx import Resource
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel, Field
 from werkzeug.exceptions import InternalServerError
 
@@ -21,7 +22,6 @@ from controllers.service_api.app.error import (
 )
 from controllers.service_api.wraps import FetchUserArg, WhereisUserArg, validate_app_token
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
-from graphon.model_runtime.errors.invoke import InvokeError
 from models.model import App, EndUser
 from services.audio_service import AudioService
 from services.errors.audio import (

api/controllers/service_api/app/completion.py

@@ -4,6 +4,7 @@ from uuid import UUID
 
 from flask import request
 from flask_restx import Resource
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel, Field, field_validator
 from werkzeug.exceptions import BadRequest, InternalServerError, NotFound
 
@@ -28,7 +29,6 @@ from core.errors.error import (
     QuotaExceededError,
 )
 from core.helper.trace_id_helper import get_external_trace_id
-from graphon.model_runtime.errors.invoke import InvokeError
 from libs import helper
 from libs.helper import UUIDStrOrEmpty
 from models.model import App, AppMode, EndUser

api/controllers/service_api/app/workflow.py

@@ -4,6 +4,9 @@ from typing import Any, Literal
 from dateutil.parser import isoparse
 from flask import request
 from flask_restx import Namespace, Resource, fields
+from graphon.enums import WorkflowExecutionStatus
+from graphon.graph_engine.manager import GraphEngineManager
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel, Field
 from sqlalchemy.orm import Session, sessionmaker
 from werkzeug.exceptions import BadRequest, InternalServerError, NotFound
@@ -30,9 +33,6 @@ from core.helper.trace_id_helper import get_external_trace_id
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from fields.workflow_app_log_fields import build_workflow_app_log_pagination_model
-from graphon.enums import WorkflowExecutionStatus
-from graphon.graph_engine.manager import GraphEngineManager
-from graphon.model_runtime.errors.invoke import InvokeError
 from libs import helper
 from libs.helper import OptionalTimestampField, TimestampField
 from models.model import App, AppMode, EndUser

api/controllers/service_api/dataset/dataset.py

@@ -2,6 +2,7 @@ from typing import Any, Literal, cast
 
 from flask import request
 from flask_restx import marshal
+from graphon.model_runtime.entities.model_entities import ModelType
 from pydantic import BaseModel, Field, TypeAdapter, field_validator
 from werkzeug.exceptions import Forbidden, NotFound
 
@@ -18,7 +19,6 @@ from core.plugin.impl.model_runtime_factory import create_plugin_provider_manage
 from core.rag.index_processor.constant.index_type import IndexTechniqueType
 from fields.dataset_fields import dataset_detail_fields
 from fields.tag_fields import DataSetTag
-from graphon.model_runtime.entities.model_entities import ModelType
 from libs.login import current_user
 from models.account import Account
 from models.dataset import DatasetPermissionEnum

api/controllers/service_api/dataset/document.py

@@ -6,7 +6,7 @@ from uuid import UUID
 from flask import request, send_file
 from flask_restx import marshal
 from pydantic import BaseModel, Field, field_validator, model_validator
-from sqlalchemy import desc, select
+from sqlalchemy import desc, func, select
 from werkzeug.exceptions import Forbidden, NotFound
 
 import services
@@ -155,7 +155,9 @@ class DocumentAddByTextApi(DatasetApiResource):
 
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.scalar(
+            select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).limit(1)
+        )
 
         if not dataset:
             raise ValueError("Dataset does not exist.")
@@ -238,7 +240,9 @@ class DocumentUpdateByTextApi(DatasetApiResource):
     def post(self, tenant_id: str, dataset_id: UUID, document_id: UUID):
         """Update document by text."""
         payload = DocumentTextUpdate.model_validate(service_api_ns.payload or {})
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == str(dataset_id)).first()
+        dataset = db.session.scalar(
+            select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == str(dataset_id)).limit(1)
+        )
         args = payload.model_dump(exclude_none=True)
         if not dataset:
             raise ValueError("Dataset does not exist.")
@@ -315,7 +319,9 @@ class DocumentAddByFileApi(DatasetApiResource):
     @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
     def post(self, tenant_id, dataset_id):
         """Create document by upload file."""
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.scalar(
+            select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).limit(1)
+        )
 
         if not dataset:
             raise ValueError("Dataset does not exist.")
@@ -425,7 +431,9 @@ class DocumentUpdateByFileApi(DatasetApiResource):
     @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
     def post(self, tenant_id, dataset_id, document_id):
         """Update document by upload file."""
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.scalar(
+            select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).limit(1)
+        )
 
         if not dataset:
             raise ValueError("Dataset does not exist.")
@@ -515,7 +523,9 @@ class DocumentListApi(DatasetApiResource):
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
         query_params = DocumentListQuery.model_validate(request.args.to_dict())
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.scalar(
+            select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).limit(1)
+        )
         if not dataset:
             raise NotFound("Dataset not found.")
 
@@ -609,7 +619,9 @@ class DocumentIndexingStatusApi(DatasetApiResource):
         batch = str(batch)
         tenant_id = str(tenant_id)
         # get dataset
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.scalar(
+            select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).limit(1)
+        )
         if not dataset:
             raise NotFound("Dataset not found.")
         # get documents
@@ -619,20 +631,23 @@ class DocumentIndexingStatusApi(DatasetApiResource):
         documents_status = []
         for document in documents:
             completed_segments = (
-                db.session.query(DocumentSegment)
-                .where(
-                    DocumentSegment.completed_at.isnot(None),
-                    DocumentSegment.document_id == str(document.id),
-                    DocumentSegment.status != SegmentStatus.RE_SEGMENT,
+                db.session.scalar(
+                    select(func.count(DocumentSegment.id)).where(
+                        DocumentSegment.completed_at.isnot(None),
+                        DocumentSegment.document_id == str(document.id),
+                        DocumentSegment.status != SegmentStatus.RE_SEGMENT,
+                    )
                 )
-                .count()
+                or 0
             )
             total_segments = (
-                db.session.query(DocumentSegment)
-                .where(
-                    DocumentSegment.document_id == str(document.id), DocumentSegment.status != SegmentStatus.RE_SEGMENT
+                db.session.scalar(
+                    select(func.count(DocumentSegment.id)).where(
+                        DocumentSegment.document_id == str(document.id),
+                        DocumentSegment.status != SegmentStatus.RE_SEGMENT,
+                    )
                 )
-                .count()
+                or 0
             )
             # Create a dictionary with document attributes and additional fields
             document_dict = {
@@ -822,7 +837,9 @@ class DocumentApi(DatasetApiResource):
         tenant_id = str(tenant_id)
 
         # get dataset info
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.scalar(
+            select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).limit(1)
+        )
 
         if not dataset:
             raise ValueError("Dataset does not exist.")

api/controllers/service_api/dataset/segment.py

@@ -2,7 +2,9 @@ from typing import Any
 
 from flask import request
 from flask_restx import marshal
+from graphon.model_runtime.entities.model_entities import ModelType
 from pydantic import BaseModel, Field
+from sqlalchemy import select
 from werkzeug.exceptions import NotFound
 
 from configs import dify_config
@@ -20,7 +22,6 @@ from core.model_manager import ModelManager
 from core.rag.index_processor.constant.index_type import IndexTechniqueType
 from extensions.ext_database import db
 from fields.segment_fields import child_chunk_fields, segment_fields
-from graphon.model_runtime.entities.model_entities import ModelType
 from libs.login import current_account_with_tenant
 from models.dataset import Dataset
 from services.dataset_service import DatasetService, DocumentService, SegmentService
@@ -28,6 +29,31 @@ from services.entities.knowledge_entities.knowledge_entities import SegmentUpdat
 from services.errors.chunk import ChildChunkDeleteIndexError, ChildChunkIndexingError
 from services.errors.chunk import ChildChunkDeleteIndexError as ChildChunkDeleteIndexServiceError
 from services.errors.chunk import ChildChunkIndexingError as ChildChunkIndexingServiceError
+from services.summary_index_service import SummaryIndexService
+
+
+def _marshal_segment_with_summary(segment, dataset_id: str) -> dict:
+    """Marshal a single segment and enrich it with summary content."""
+    segment_dict = dict(marshal(segment, segment_fields))  # type: ignore[arg-type]
+    summary = SummaryIndexService.get_segment_summary(segment_id=segment.id, dataset_id=dataset_id)
+    segment_dict["summary"] = summary.summary_content if summary else None
+    return segment_dict
+
+
+def _marshal_segments_with_summary(segments, dataset_id: str) -> list[dict]:
+    """Marshal multiple segments and enrich them with summary content (batch query)."""
+    segment_ids = [segment.id for segment in segments]
+    summaries: dict = {}
+    if segment_ids:
+        summary_records = SummaryIndexService.get_segments_summaries(segment_ids=segment_ids, dataset_id=dataset_id)
+        summaries = {chunk_id: record.summary_content for chunk_id, record in summary_records.items()}
+
+    result = []
+    for segment in segments:
+        segment_dict = dict(marshal(segment, segment_fields))  # type: ignore[arg-type]
+        segment_dict["summary"] = summaries.get(segment.id)
+        result.append(segment_dict)
+    return result
 
 
 class SegmentCreatePayload(BaseModel):
@@ -92,7 +118,9 @@ class SegmentApi(DatasetApiResource):
         _, current_tenant_id = current_account_with_tenant()
         """Create single segment."""
         # check dataset
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.scalar(
+            select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).limit(1)
+        )
         if not dataset:
             raise NotFound("Dataset not found.")
         # check document
@@ -129,7 +157,7 @@ class SegmentApi(DatasetApiResource):
             for args_item in payload.segments:
                 SegmentService.segment_create_args_validate(args_item, document)
             segments = SegmentService.multi_create_segment(payload.segments, document, dataset)
-            return {"data": marshal(segments, segment_fields), "doc_form": document.doc_form}, 200
+            return {"data": _marshal_segments_with_summary(segments, dataset_id), "doc_form": document.doc_form}, 200
         else:
             return {"error": "Segments is required"}, 400
 
@@ -150,7 +178,9 @@ class SegmentApi(DatasetApiResource):
         # check dataset
         page = request.args.get("page", default=1, type=int)
         limit = request.args.get("limit", default=20, type=int)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.scalar(
+            select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).limit(1)
+        )
         if not dataset:
             raise NotFound("Dataset not found.")
         # check document
@@ -191,7 +221,7 @@ class SegmentApi(DatasetApiResource):
         )
 
         response = {
-            "data": marshal(segments, segment_fields),
+            "data": _marshal_segments_with_summary(segments, dataset_id),
             "doc_form": document.doc_form,
             "total": total,
             "has_more": len(segments) == limit,
@@ -220,7 +250,9 @@ class DatasetSegmentApi(DatasetApiResource):
     def delete(self, tenant_id: str, dataset_id: str, document_id: str, segment_id: str):
         _, current_tenant_id = current_account_with_tenant()
         # check dataset
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.scalar(
+            select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).limit(1)
+        )
         if not dataset:
             raise NotFound("Dataset not found.")
         # check user's model setting
@@ -254,7 +286,9 @@ class DatasetSegmentApi(DatasetApiResource):
     def post(self, tenant_id: str, dataset_id: str, document_id: str, segment_id: str):
         _, current_tenant_id = current_account_with_tenant()
         # check dataset
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.scalar(
+            select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).limit(1)
+        )
         if not dataset:
             raise NotFound("Dataset not found.")
         # check user's model setting
@@ -287,7 +321,7 @@ class DatasetSegmentApi(DatasetApiResource):
         payload = SegmentUpdatePayload.model_validate(service_api_ns.payload or {})
 
         updated_segment = SegmentService.update_segment(payload.segment, segment, document, dataset)
-        return {"data": marshal(updated_segment, segment_fields), "doc_form": document.doc_form}, 200
+        return {"data": _marshal_segment_with_summary(updated_segment, dataset_id), "doc_form": document.doc_form}, 200
 
     @service_api_ns.doc("get_segment")
     @service_api_ns.doc(description="Get a specific segment by ID")
@@ -301,7 +335,9 @@ class DatasetSegmentApi(DatasetApiResource):
     def get(self, tenant_id: str, dataset_id: str, document_id: str, segment_id: str):
         _, current_tenant_id = current_account_with_tenant()
         # check dataset
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.scalar(
+            select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).limit(1)
+        )
         if not dataset:
             raise NotFound("Dataset not found.")
         # check user's model setting
@@ -315,7 +351,7 @@ class DatasetSegmentApi(DatasetApiResource):
         if not segment:
             raise NotFound("Segment not found.")
 
-        return {"data": marshal(segment, segment_fields), "doc_form": document.doc_form}, 200
+        return {"data": _marshal_segment_with_summary(segment, dataset_id), "doc_form": document.doc_form}, 200
 
 
 @service_api_ns.route(
@@ -344,7 +380,9 @@ class ChildChunkApi(DatasetApiResource):
         _, current_tenant_id = current_account_with_tenant()
         """Create child chunk."""
         # check dataset
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.scalar(
+            select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).limit(1)
+        )
         if not dataset:
             raise NotFound("Dataset not found.")
 
@@ -402,7 +440,9 @@ class ChildChunkApi(DatasetApiResource):
         _, current_tenant_id = current_account_with_tenant()
         """Get child chunks."""
         # check dataset
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.scalar(
+            select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).limit(1)
+        )
         if not dataset:
             raise NotFound("Dataset not found.")
 
@@ -468,7 +508,9 @@ class DatasetChildChunkApi(DatasetApiResource):
         _, current_tenant_id = current_account_with_tenant()
         """Delete child chunk."""
         # check dataset
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.scalar(
+            select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).limit(1)
+        )
         if not dataset:
             raise NotFound("Dataset not found.")
 
@@ -527,7 +569,9 @@ class DatasetChildChunkApi(DatasetApiResource):
         _, current_tenant_id = current_account_with_tenant()
         """Update child chunk."""
         # check dataset
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.scalar(
+            select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).limit(1)
+        )
         if not dataset:
             raise NotFound("Dataset not found.")
 

api/controllers/service_api/workspace/models.py

@@ -1,9 +1,9 @@
 from flask_login import current_user
 from flask_restx import Resource
+from graphon.model_runtime.utils.encoders import jsonable_encoder
 
 from controllers.service_api import service_api_ns
 from controllers.service_api.wraps import validate_dataset_token
-from graphon.model_runtime.utils.encoders import jsonable_encoder
 from services.model_provider_service import ModelProviderService
 
 

api/controllers/web/audio.py

@@ -2,6 +2,7 @@ import logging
 
 from flask import request
 from flask_restx import fields, marshal_with
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel, field_validator
 from werkzeug.exceptions import InternalServerError
 
@@ -20,7 +21,6 @@ from controllers.web.error import (
 )
 from controllers.web.wraps import WebApiResource
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
-from graphon.model_runtime.errors.invoke import InvokeError
 from libs.helper import uuid_value
 from models.model import App
 from services.audio_service import AudioService

api/controllers/web/completion.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Any, Literal
 
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel, Field, field_validator
 from werkzeug.exceptions import InternalServerError, NotFound
 
@@ -25,7 +26,6 @@ from core.errors.error import (
     ProviderTokenNotInitError,
     QuotaExceededError,
 )
-from graphon.model_runtime.errors.invoke import InvokeError
 from libs import helper
 from libs.helper import uuid_value
 from models.model import AppMode

api/controllers/web/message.py

@@ -2,6 +2,7 @@ import logging
 from typing import Literal
 
 from flask import request
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel, Field, TypeAdapter, field_validator
 from werkzeug.exceptions import InternalServerError, NotFound
 
@@ -22,7 +23,6 @@ from core.app.entities.app_invoke_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from fields.conversation_fields import ResultResponse
 from fields.message_fields import SuggestedQuestionsResponse, WebMessageInfiniteScrollPagination, WebMessageListItem
-from graphon.model_runtime.errors.invoke import InvokeError
 from libs import helper
 from libs.helper import uuid_value
 from models.enums import FeedbackRating

api/controllers/web/remote_files.py

@@ -1,6 +1,7 @@
 import urllib.parse
 
 import httpx
+from graphon.file import helpers as file_helpers
 from pydantic import BaseModel, Field, HttpUrl
 
 import services
@@ -13,7 +14,6 @@ from controllers.common.errors import (
 from core.helper import ssrf_proxy
 from extensions.ext_database import db
 from fields.file_fields import FileWithSignedUrl, RemoteFileInfo
-from graphon.file import helpers as file_helpers
 from services.file_service import FileService
 
 from ..common.schema import register_schema_models

api/controllers/web/workflow.py

@@ -1,6 +1,8 @@
 import logging
 from typing import Any
 
+from graphon.graph_engine.manager import GraphEngineManager
+from graphon.model_runtime.errors.invoke import InvokeError
 from pydantic import BaseModel, Field
 from werkzeug.exceptions import InternalServerError
 
@@ -23,8 +25,6 @@ from core.errors.error import (
     QuotaExceededError,
 )
 from extensions.ext_redis import redis_client
-from graphon.graph_engine.manager import GraphEngineManager
-from graphon.model_runtime.errors.invoke import InvokeError
 from libs import helper
 from models.model import App, AppMode, EndUser
 from services.app_generate_service import AppGenerateService

api/core/agent/base_agent_runner.py

@@ -4,7 +4,21 @@ import uuid
 from decimal import Decimal
 from typing import Union, cast
 
-from sqlalchemy import select
+from graphon.file import file_manager
+from graphon.model_runtime.entities import (
+    AssistantPromptMessage,
+    LLMUsage,
+    PromptMessage,
+    PromptMessageTool,
+    SystemPromptMessage,
+    TextPromptMessageContent,
+    ToolPromptMessage,
+    UserPromptMessage,
+)
+from graphon.model_runtime.entities.message_entities import ImagePromptMessageContent, PromptMessageContentUnionTypes
+from graphon.model_runtime.entities.model_entities import ModelFeature
+from graphon.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+from sqlalchemy import func, select
 
 from core.agent.entities import AgentEntity, AgentToolEntity
 from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
@@ -29,20 +43,6 @@ from core.tools.tool_manager import ToolManager
 from core.tools.utils.dataset_retriever_tool import DatasetRetrieverTool
 from extensions.ext_database import db
 from factories import file_factory
-from graphon.file import file_manager
-from graphon.model_runtime.entities import (
-    AssistantPromptMessage,
-    LLMUsage,
-    PromptMessage,
-    PromptMessageTool,
-    SystemPromptMessage,
-    TextPromptMessageContent,
-    ToolPromptMessage,
-    UserPromptMessage,
-)
-from graphon.model_runtime.entities.message_entities import ImagePromptMessageContent, PromptMessageContentUnionTypes
-from graphon.model_runtime.entities.model_entities import ModelFeature
-from graphon.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from models.enums import CreatorUserRole
 from models.model import Conversation, Message, MessageAgentThought, MessageFile
 
@@ -104,11 +104,14 @@ class BaseAgentRunner(AppRunner):
         )
         # get how many agent thoughts have been created
         self.agent_thought_count = (
-            db.session.query(MessageAgentThought)
-            .where(
-                MessageAgentThought.message_id == self.message.id,
+            db.session.scalar(
+                select(func.count())
+                .select_from(MessageAgentThought)
+                .where(
+                    MessageAgentThought.message_id == self.message.id,
+                )
             )
-            .count()
+            or 0
         )
         db.session.close()
 

api/core/agent/cot_agent_runner.py

@@ -4,6 +4,15 @@ from abc import ABC, abstractmethod
 from collections.abc import Generator, Mapping, Sequence
 from typing import Any
 
+from graphon.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
+from graphon.model_runtime.entities.message_entities import (
+    AssistantPromptMessage,
+    PromptMessage,
+    PromptMessageTool,
+    ToolPromptMessage,
+    UserPromptMessage,
+)
+
 from core.agent.base_agent_runner import BaseAgentRunner
 from core.agent.entities import AgentScratchpadUnit
 from core.agent.errors import AgentMaxIterationError
@@ -15,14 +24,6 @@ from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransfo
 from core.tools.__base.tool import Tool
 from core.tools.entities.tool_entities import ToolInvokeMeta
 from core.tools.tool_engine import ToolEngine
-from graphon.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
-from graphon.model_runtime.entities.message_entities import (
-    AssistantPromptMessage,
-    PromptMessage,
-    PromptMessageTool,
-    ToolPromptMessage,
-    UserPromptMessage,
-)
 from models.model import Message
 
 logger = logging.getLogger(__name__)

api/core/agent/cot_chat_agent_runner.py

@@ -1,6 +1,5 @@
 import json
 
-from core.agent.cot_agent_runner import CotAgentRunner
 from graphon.file import file_manager
 from graphon.model_runtime.entities import (
     AssistantPromptMessage,
@@ -12,6 +11,8 @@ from graphon.model_runtime.entities import (
 from graphon.model_runtime.entities.message_entities import ImagePromptMessageContent, PromptMessageContentUnionTypes
 from graphon.model_runtime.utils.encoders import jsonable_encoder
 
+from core.agent.cot_agent_runner import CotAgentRunner
+
 
 class CotChatAgentRunner(CotAgentRunner):
     def _organize_system_prompt(self) -> SystemPromptMessage:

api/core/agent/cot_completion_agent_runner.py

@@ -1,6 +1,5 @@
 import json
 
-from core.agent.cot_agent_runner import CotAgentRunner
 from graphon.model_runtime.entities.message_entities import (
     AssistantPromptMessage,
     PromptMessage,
@@ -9,6 +8,8 @@ from graphon.model_runtime.entities.message_entities import (
 )
 from graphon.model_runtime.utils.encoders import jsonable_encoder
 
+from core.agent.cot_agent_runner import CotAgentRunner
+
 
 class CotCompletionAgentRunner(CotAgentRunner):
     def _organize_instruction_prompt(self) -> str:

api/core/agent/fc_agent_runner.py

@@ -4,13 +4,6 @@ from collections.abc import Generator
 from copy import deepcopy
 from typing import Any, Union
 
-from core.agent.base_agent_runner import BaseAgentRunner
-from core.agent.errors import AgentMaxIterationError
-from core.app.apps.base_app_queue_manager import PublishFrom
-from core.app.entities.queue_entities import QueueAgentThoughtEvent, QueueMessageEndEvent, QueueMessageFileEvent
-from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransform
-from core.tools.entities.tool_entities import ToolInvokeMeta
-from core.tools.tool_engine import ToolEngine
 from graphon.file import file_manager
 from graphon.model_runtime.entities import (
     AssistantPromptMessage,
@@ -26,6 +19,14 @@ from graphon.model_runtime.entities import (
     UserPromptMessage,
 )
 from graphon.model_runtime.entities.message_entities import ImagePromptMessageContent, PromptMessageContentUnionTypes
+
+from core.agent.base_agent_runner import BaseAgentRunner
+from core.agent.errors import AgentMaxIterationError
+from core.app.apps.base_app_queue_manager import PublishFrom
+from core.app.entities.queue_entities import QueueAgentThoughtEvent, QueueMessageEndEvent, QueueMessageFileEvent
+from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransform
+from core.tools.entities.tool_entities import ToolInvokeMeta
+from core.tools.tool_engine import ToolEngine
 from models.model import Message
 
 logger = logging.getLogger(__name__)

api/core/agent/output_parser/cot_output_parser.py

@@ -3,9 +3,10 @@ import re
 from collections.abc import Generator
 from typing import Union
 
-from core.agent.entities import AgentScratchpadUnit
 from graphon.model_runtime.entities.llm_entities import LLMResultChunk
 
+from core.agent.entities import AgentScratchpadUnit
+
 
 class CotAgentOutputParser:
     @classmethod

api/core/app/app_config/easy_ui_based_app/model_config/converter.py

@@ -1,13 +1,14 @@
 from typing import cast
 
+from graphon.model_runtime.entities.llm_entities import LLMMode
+from graphon.model_runtime.entities.model_entities import ModelPropertyKey, ModelType
+from graphon.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+
 from core.app.app_config.entities import EasyUIBasedAppConfig
 from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
 from core.entities.model_entities import ModelStatus
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager
-from graphon.model_runtime.entities.llm_entities import LLMMode
-from graphon.model_runtime.entities.model_entities import ModelPropertyKey, ModelType
-from graphon.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 
 
 class ModelConfigConverter:

api/core/app/app_config/easy_ui_based_app/model_config/manager.py

@@ -1,9 +1,10 @@
 from collections.abc import Mapping
 from typing import Any
 
+from graphon.model_runtime.entities.model_entities import ModelPropertyKey, ModelType
+
 from core.app.app_config.entities import ModelConfigEntity
 from core.plugin.impl.model_runtime_factory import create_plugin_model_assembly
-from graphon.model_runtime.entities.model_entities import ModelPropertyKey, ModelType
 from models.model import AppModelConfigDict
 from models.provider_ids import ModelProviderID
 

api/core/app/app_config/easy_ui_based_app/prompt_template/manager.py

@@ -1,5 +1,7 @@
 from typing import Any
 
+from graphon.model_runtime.entities.message_entities import PromptMessageRole
+
 from core.app.app_config.entities import (
     AdvancedChatMessageEntity,
     AdvancedChatPromptTemplateEntity,
@@ -7,7 +9,6 @@ from core.app.app_config.entities import (
     PromptTemplateEntity,
 )
 from core.prompt.simple_prompt_transform import ModelMode
-from graphon.model_runtime.entities.message_entities import PromptMessageRole
 from models.model import AppMode, AppModelConfigDict
 
 

api/core/app/app_config/easy_ui_based_app/variables/manager.py

@@ -1,9 +1,10 @@
 import re
 from typing import cast
 
+from graphon.variables.input_entities import VariableEntity, VariableEntityType
+
 from core.app.app_config.entities import ExternalDataVariableEntity
 from core.external_data_tool.factory import ExternalDataToolFactory
-from graphon.variables.input_entities import VariableEntity, VariableEntityType
 from models.model import AppModelConfigDict
 
 _ALLOWED_VARIABLE_ENTITY_TYPE = frozenset(

api/core/app/app_config/entities.py

@@ -2,13 +2,13 @@ from collections.abc import Sequence
 from enum import StrEnum, auto
 from typing import Any, Literal
 
-from pydantic import BaseModel, Field
-
-from core.rag.data_post_processor.data_post_processor import RerankingModelDict, WeightsDict
 from graphon.file import FileUploadConfig
 from graphon.model_runtime.entities.llm_entities import LLMMode
 from graphon.model_runtime.entities.message_entities import PromptMessageRole
 from graphon.variables.input_entities import VariableEntity as WorkflowVariableEntity
+from pydantic import BaseModel, Field
+
+from core.rag.data_post_processor.data_post_processor import RerankingModelDict, WeightsDict
 from models.model import AppMode
 
 

api/core/app/app_config/features/file_upload/manager.py

@@ -1,9 +1,10 @@
 from collections.abc import Mapping
 from typing import Any
 
-from constants import DEFAULT_FILE_NUMBER_LIMITS
 from graphon.file import FileUploadConfig
 
+from constants import DEFAULT_FILE_NUMBER_LIMITS
+
 
 class FileUploadConfigManager:
     @classmethod

api/core/app/app_config/workflow_ui_based_app/variables/manager.py

@@ -1,7 +1,8 @@
 import re
 
-from core.app.app_config.entities import RagPipelineVariableEntity
 from graphon.variables.input_entities import VariableEntity
+
+from core.app.app_config.entities import RagPipelineVariableEntity
 from models.workflow import Workflow
 
 

api/core/app/apps/advanced_chat/app_generator.py

@@ -5,7 +5,7 @@ import logging
 import threading
 import uuid
 from collections.abc import Generator, Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Literal, TypeVar, Union, overload
+from typing import TYPE_CHECKING, Any, Literal, Union, overload
 
 from flask import Flask, current_app
 from pydantic import ValidationError
@@ -18,11 +18,21 @@ from constants import UUID_NIL
 
 if TYPE_CHECKING:
     from controllers.console.app.workflow import LoopNodeRunPayload
+from graphon.graph_engine.layers import GraphEngineLayer
+from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
+from graphon.runtime import GraphRuntimeState
+from graphon.variable_loader import DUMMY_VARIABLE_LOADER, VariableLoader
+
 from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
 from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager
 from core.app.apps.advanced_chat.app_runner import AdvancedChatAppRunner
 from core.app.apps.advanced_chat.generate_response_converter import AdvancedChatAppGenerateResponseConverter
-from core.app.apps.advanced_chat.generate_task_pipeline import AdvancedChatAppGenerateTaskPipeline
+from core.app.apps.advanced_chat.generate_task_pipeline import (
+    AdvancedChatAppGenerateTaskPipeline,
+    ConversationSnapshot,
+    MessageSnapshot,
+    WorkflowSnapshot,
+)
 from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
 from core.app.apps.draft_variable_saver import DraftVariableSaverFactory
 from core.app.apps.exc import GenerateTaskStoppedError
@@ -38,13 +48,8 @@ from core.repositories import DifyCoreRepositoryFactory
 from core.repositories.factory import WorkflowExecutionRepository, WorkflowNodeExecutionRepository
 from extensions.ext_database import db
 from factories import file_factory
-from graphon.graph_engine.layers.base import GraphEngineLayer
-from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
-from graphon.runtime import GraphRuntimeState
-from graphon.variable_loader import DUMMY_VARIABLE_LOADER, VariableLoader
 from libs.flask_utils import preserve_flask_contexts
 from models import Account, App, Conversation, EndUser, Message, Workflow, WorkflowNodeExecutionTriggeredFrom
-from models.base import Base
 from models.enums import WorkflowRunTriggeredFrom
 from services.conversation_service import ConversationService
 from services.workflow_draft_variable_service import (
@@ -524,19 +529,20 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
 
             worker_thread.start()
 
-            # release database connection, because the following new thread operations may take a long time
-            with Session(bind=db.engine, expire_on_commit=False) as session:
-                workflow = _refresh_model(session, workflow)
-                message = _refresh_model(session, message)
+            # Capture the scalar fields needed by the response pipeline before
+            # releasing the request-scoped SQLAlchemy session.
+            workflow_snapshot = WorkflowSnapshot.from_workflow(workflow)
+            conversation_snapshot = ConversationSnapshot.from_conversation(conversation)
+            message_snapshot = MessageSnapshot.from_message(message)
             db.session.close()
 
             # return response or stream generator
             response = self._handle_advanced_chat_response(
                 application_generate_entity=application_generate_entity,
-                workflow=workflow,
+                workflow=workflow_snapshot,
                 queue_manager=queue_manager,
-                conversation=conversation,
-                message=message,
+                conversation=conversation_snapshot,
+                message=message_snapshot,
                 user=user,
                 stream=stream,
                 draft_var_saver_factory=self._get_draft_var_saver_factory(invoke_from, account=user),
@@ -643,10 +649,10 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
         self,
         *,
         application_generate_entity: AdvancedChatAppGenerateEntity,
-        workflow: Workflow,
+        workflow: WorkflowSnapshot,
         queue_manager: AppQueueManager,
-        conversation: Conversation,
-        message: Message,
+        conversation: ConversationSnapshot,
+        message: MessageSnapshot,
         user: Union[Account, EndUser],
         draft_var_saver_factory: DraftVariableSaverFactory,
         stream: bool = False,
@@ -683,13 +689,3 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             else:
                 logger.exception("Failed to process generate task pipeline, conversation_id: %s", conversation.id)
                 raise e
-
-
-_T = TypeVar("_T", bound=Base)
-
-
-def _refresh_model(session, model: _T) -> _T:
-    with Session(bind=db.engine, expire_on_commit=False) as session:
-        detach_model = session.get(type(model), model.id)
-        assert detach_model is not None
-        return detach_model

api/core/app/apps/advanced_chat/app_runner.py

@@ -3,6 +3,12 @@ import time
 from collections.abc import Mapping, Sequence
 from typing import Any, cast
 
+from graphon.enums import WorkflowType
+from graphon.graph_engine.command_channels import RedisChannel
+from graphon.graph_engine.layers import GraphEngineLayer
+from graphon.runtime import GraphRuntimeState, VariablePool
+from graphon.variable_loader import VariableLoader
+from graphon.variables.variables import Variable
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
@@ -37,12 +43,6 @@ from core.workflow.workflow_entry import WorkflowEntry
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from extensions.otel import WorkflowAppRunnerHandler, trace_span
-from graphon.enums import WorkflowType
-from graphon.graph_engine.command_channels.redis_channel import RedisChannel
-from graphon.graph_engine.layers.base import GraphEngineLayer
-from graphon.runtime import GraphRuntimeState, VariablePool
-from graphon.variable_loader import VariableLoader
-from graphon.variables.variables import Variable
 from models import Workflow
 from models.model import App, Conversation, Message, MessageAnnotation
 from models.workflow import ConversationVariable
@@ -137,6 +137,7 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
                 workflow=self._workflow,
                 single_iteration_run=self.application_generate_entity.single_iteration_run,
                 single_loop_run=self.application_generate_entity.single_loop_run,
+                user_id=self.application_generate_entity.user_id,
             )
         else:
             inputs = self.application_generate_entity.inputs

api/core/app/apps/advanced_chat/generate_task_pipeline.py

@@ -4,9 +4,17 @@ import re
 import time
 from collections.abc import Callable, Generator, Mapping
 from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import datetime
 from threading import Thread
 from typing import Any, Union
 
+from graphon.entities.pause_reason import HumanInputRequired
+from graphon.enums import WorkflowExecutionStatus
+from graphon.model_runtime.entities.llm_entities import LLMUsage
+from graphon.model_runtime.utils.encoders import jsonable_encoder
+from graphon.nodes import BuiltinNodeTypes
+from graphon.runtime import GraphRuntimeState
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
@@ -69,21 +77,63 @@ from core.repositories.human_input_repository import HumanInputFormRepositoryImp
 from core.workflow.file_reference import resolve_file_record_id
 from core.workflow.system_variables import build_system_variables
 from extensions.ext_database import db
-from graphon.entities.pause_reason import HumanInputRequired
-from graphon.enums import WorkflowExecutionStatus
-from graphon.model_runtime.entities.llm_entities import LLMUsage
-from graphon.model_runtime.utils.encoders import jsonable_encoder
-from graphon.nodes import BuiltinNodeTypes
-from graphon.runtime import GraphRuntimeState
 from libs.datetime_utils import naive_utc_now
 from models import Account, Conversation, EndUser, Message, MessageFile
 from models.enums import CreatorUserRole, MessageFileBelongsTo, MessageStatus
 from models.execution_extra_content import HumanInputContent
+from models.model import AppMode
 from models.workflow import Workflow
 
 logger = logging.getLogger(__name__)
 
 
+@dataclass(frozen=True, slots=True)
+class WorkflowSnapshot:
+    id: str
+    tenant_id: str
+    features_dict: Mapping[str, Any]
+
+    @classmethod
+    def from_workflow(cls, workflow: Workflow) -> "WorkflowSnapshot":
+        return cls(
+            id=workflow.id,
+            tenant_id=workflow.tenant_id,
+            features_dict=dict(workflow.features_dict),
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class ConversationSnapshot:
+    id: str
+    mode: AppMode
+
+    @classmethod
+    def from_conversation(cls, conversation: Conversation) -> "ConversationSnapshot":
+        return cls(
+            id=conversation.id,
+            mode=conversation.mode,
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class MessageSnapshot:
+    id: str
+    query: str
+    created_at: datetime
+    status: MessageStatus
+    answer: str
+
+    @classmethod
+    def from_message(cls, message: Message) -> "MessageSnapshot":
+        return cls(
+            id=message.id,
+            query=message.query,
+            created_at=message.created_at,
+            status=message.status,
+            answer=message.answer,
+        )
+
+
 class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
     """
     AdvancedChatAppGenerateTaskPipeline is a class that generate stream output and state management for Application.
@@ -92,10 +142,10 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
     def __init__(
         self,
         application_generate_entity: AdvancedChatAppGenerateEntity,
-        workflow: Workflow,
+        workflow: WorkflowSnapshot,
         queue_manager: AppQueueManager,
-        conversation: Conversation,
-        message: Message,
+        conversation: ConversationSnapshot,
+        message: MessageSnapshot,
         user: Union[Account, EndUser],
         stream: bool,
         dialogue_count: int,
@@ -156,7 +206,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
         self._message_saved_on_pause = False
         self._seed_graph_runtime_state_from_queue_manager()
 
-    def _seed_task_state_from_message(self, message: Message) -> None:
+    def _seed_task_state_from_message(self, message: MessageSnapshot) -> None:
         if message.status == MessageStatus.PAUSED and message.answer:
             self._task_state.answer = message.answer
 

api/core/app/apps/agent_chat/app_generator.py

@@ -6,6 +6,7 @@ from collections.abc import Generator, Mapping
 from typing import Any, Literal, Union, overload
 
 from flask import Flask, current_app
+from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
 from pydantic import ValidationError
 
 from configs import dify_config
@@ -23,7 +24,6 @@ from core.app.entities.app_invoke_entities import AgentChatAppGenerateEntity, In
 from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
 from factories import file_factory
-from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
 from libs.flask_utils import preserve_flask_contexts
 from models import Account, App, EndUser
 from services.conversation_service import ConversationService

api/core/app/apps/agent_chat/app_runner.py

@@ -1,6 +1,9 @@
 import logging
 from typing import cast
 
+from graphon.model_runtime.entities.llm_entities import LLMMode
+from graphon.model_runtime.entities.model_entities import ModelFeature, ModelPropertyKey
+from graphon.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from sqlalchemy import select
 
 from core.agent.cot_chat_agent_runner import CotChatAgentRunner
@@ -16,9 +19,6 @@ from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_manager import ModelInstance
 from core.moderation.base import ModerationError
 from extensions.ext_database import db
-from graphon.model_runtime.entities.llm_entities import LLMMode
-from graphon.model_runtime.entities.model_entities import ModelFeature, ModelPropertyKey
-from graphon.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from models.model import App, Conversation, Message
 
 logger = logging.getLogger(__name__)

api/core/app/apps/base_app_generate_response_converter.py

@@ -3,10 +3,11 @@ from abc import ABC, abstractmethod
 from collections.abc import Generator, Mapping
 from typing import Any, Union
 
+from graphon.model_runtime.errors.invoke import InvokeError
+
 from core.app.entities.app_invoke_entities import InvokeFrom
 from core.app.entities.task_entities import AppBlockingResponse, AppStreamResponse
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
-from graphon.model_runtime.errors.invoke import InvokeError
 
 logger = logging.getLogger(__name__)
 

api/core/app/apps/base_app_generator.py

@@ -2,6 +2,9 @@ from collections.abc import Generator, Mapping, Sequence
 from contextlib import AbstractContextManager, nullcontext
 from typing import TYPE_CHECKING, Any, Union, final
 
+from graphon.enums import NodeType
+from graphon.file import File, FileUploadConfig
+from graphon.variables.input_entities import VariableEntityType
 from sqlalchemy.orm import Session
 
 from core.app.apps.draft_variable_saver import (
@@ -13,9 +16,6 @@ from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom
 from core.app.file_access import DatabaseFileAccessController, FileAccessScope, bind_file_access_scope
 from extensions.ext_database import db
 from factories import file_factory
-from graphon.enums import NodeType
-from graphon.file import File, FileUploadConfig
-from graphon.variables.input_entities import VariableEntityType
 from libs.orjson import orjson_dumps
 from models import Account, EndUser
 from services.workflow_draft_variable_service import DraftVariableSaver as DraftVariableSaverImpl

api/core/app/apps/base_app_queue_manager.py

@@ -7,6 +7,7 @@ from enum import IntEnum, auto
 from typing import Any
 
 from cachetools import TTLCache, cachedmethod
+from graphon.runtime import GraphRuntimeState
 from redis.exceptions import RedisError
 from sqlalchemy.orm import DeclarativeMeta
 
@@ -21,7 +22,6 @@ from core.app.entities.queue_entities import (
     WorkflowQueueMessage,
 )
 from extensions.ext_redis import redis_client
-from graphon.runtime import GraphRuntimeState
 
 logger = logging.getLogger(__name__)
 

api/core/app/apps/base_app_runner.py

@@ -5,6 +5,17 @@ from collections.abc import Generator, Mapping, Sequence
 from mimetypes import guess_extension
 from typing import TYPE_CHECKING, Any, Union
 
+from graphon.file import FileTransferMethod, FileType
+from graphon.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
+from graphon.model_runtime.entities.message_entities import (
+    AssistantPromptMessage,
+    ImagePromptMessageContent,
+    PromptMessage,
+    TextPromptMessageContent,
+)
+from graphon.model_runtime.entities.model_entities import ModelPropertyKey
+from graphon.model_runtime.errors.invoke import InvokeBadRequestError
+
 from core.app.app_config.entities import ExternalDataVariableEntity, PromptTemplateEntity
 from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
 from core.app.entities.app_invoke_entities import (
@@ -30,21 +41,11 @@ from core.prompt.entities.advanced_prompt_entities import ChatModelMessage, Comp
 from core.prompt.simple_prompt_transform import ModelMode, SimplePromptTransform
 from core.tools.tool_file_manager import ToolFileManager
 from extensions.ext_database import db
-from graphon.file.enums import FileTransferMethod, FileType
-from graphon.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
-from graphon.model_runtime.entities.message_entities import (
-    AssistantPromptMessage,
-    ImagePromptMessageContent,
-    PromptMessage,
-    TextPromptMessageContent,
-)
-from graphon.model_runtime.entities.model_entities import ModelPropertyKey
-from graphon.model_runtime.errors.invoke import InvokeBadRequestError
 from models.enums import CreatorUserRole, MessageFileBelongsTo
 from models.model import App, AppMode, Message, MessageAnnotation, MessageFile
 
 if TYPE_CHECKING:
-    from graphon.file.models import File
+    from graphon.file import File
 
 _logger = logging.getLogger(__name__)
 

api/core/app/apps/chat/app_generator.py

@@ -6,6 +6,7 @@ from collections.abc import Generator, Mapping
 from typing import Any, Literal, Union, overload
 
 from flask import Flask, copy_current_request_context, current_app
+from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
 from pydantic import ValidationError
 
 from configs import dify_config
@@ -23,7 +24,6 @@ from core.app.entities.app_invoke_entities import ChatAppGenerateEntity, InvokeF
 from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
 from factories import file_factory
-from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
 from models import Account
 from models.model import App, EndUser
 from services.conversation_service import ConversationService

api/core/app/apps/chat/app_runner.py

@@ -1,6 +1,8 @@
 import logging
 from typing import cast
 
+from graphon.file import File
+from graphon.model_runtime.entities.message_entities import ImagePromptMessageContent
 from sqlalchemy import select
 
 from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
@@ -16,8 +18,6 @@ from core.model_manager import ModelInstance
 from core.moderation.base import ModerationError
 from core.rag.retrieval.dataset_retrieval import DatasetRetrieval
 from extensions.ext_database import db
-from graphon.file import File
-from graphon.model_runtime.entities.message_entities import ImagePromptMessageContent
 from models.model import App, Conversation, Message
 
 logger = logging.getLogger(__name__)