bump to 0.6.12 (#5712)

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.devcontainer/post_create_command.sh

@@ -3,7 +3,7 @@
 cd web && npm install
 
 echo 'alias start-api="cd /workspaces/dify/api && flask run --host 0.0.0.0 --port=5001 --debug"' >> ~/.bashrc
-echo 'alias start-worker="cd /workspaces/dify/api && celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail"' >> ~/.bashrc
+echo 'alias start-worker="cd /workspaces/dify/api && celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace"' >> ~/.bashrc
 echo 'alias start-web="cd /workspaces/dify/web && npm run dev"' >> ~/.bashrc
 echo 'alias start-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify up -d"' >> ~/.bashrc
 

.devcontainer/post_start_command.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-cd api && pip install -r requirements.txt
+poetry install -C api

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -14,13 +14,15 @@ body:
           required: true
         - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
+        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
+          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
 
   - type: input
     attributes:
       label: Dify version
-      placeholder: 0.3.21
+      placeholder: 0.6.11
       description: See about section in Dify console
     validations:
       required: true
@@ -40,7 +42,7 @@ body:
   - type: textarea
     attributes:
       label: Steps to reproduce
-      description: We highly suggest including screenshots and a bug report log.
+      description: We highly suggest including screenshots and a bug report log. Please use the right markdown syntax for code blocks.
       placeholder: Having detailed steps helps us reproduce the bug.
     validations:
       required: true

.github/ISSUE_TEMPLATE/document_issue.yml

@@ -12,6 +12,8 @@ body:
           required: true
         - label: I confirm that I am using English to submit report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
+        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
+          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: textarea

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -12,35 +12,25 @@ body:
           required: true
         - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
+        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
+          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: textarea
     attributes:
-      label: 1. Is this request related to a challenge you're experiencing?
+      label: 1. Is this request related to a challenge you're experiencing? Tell me about your story.
       placeholder: Please describe the specific scenario or problem you're facing as clearly as possible. For instance "I was trying to use [feature] for [specific task], and [what happened]... It was frustrating because...."
     validations:
       required: true
   - type: textarea
     attributes:
-      label: 2. Describe the feature you'd like to see
-      placeholder: Think about what you want to achieve and how this feature will help you. Sketches, flow diagrams, or any visual representation will be a major plus.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: 3. How will this feature improve your workflow or experience?
-      placeholder: Tell us how this change will benefit your work. This helps us prioritize based on user impact.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: 4. Additional context or comments
+      label: 2. Additional context or comments
       placeholder: (Any other information, comments, documentations, links, or screenshots that would provide more clarity. This is the place to add anything else not covered above.)
     validations:
       required: false
   - type: checkboxes
     attributes:
-      label: 5. Can you help us with this feature?
+      label: 3. Can you help us with this feature?
       description: Let us know! This is not a commitment, but a starting point for collaboration.
       options:
         - label: I am interested in contributing to this feature.

.github/ISSUE_TEMPLATE/translation_issue.yml

@@ -12,6 +12,8 @@ body:
           required: true
         - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
+        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
+          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: input

.github/workflows/api-tests.yml

@@ -14,74 +14,6 @@ concurrency:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version:
-          - "3.10"
-          - "3.11"
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: |
-            ./api/requirements.txt
-            ./api/requirements-dev.txt
-
-      - name: Install dependencies
-        run: pip install -r ./api/requirements.txt -r ./api/requirements-dev.txt
-
-      - name: Run Unit tests
-        run: dev/pytest/pytest_unit_tests.sh
-
-      - name: Run ModelRuntime
-        run: dev/pytest/pytest_model_runtime.sh
-
-      - name: Run Tool
-        run: dev/pytest/pytest_tools.sh
-
-      - name: Set up Sandbox
-        uses: hoverkraft-tech/compose-action@v2.0.0
-        with:
-          compose-file: |
-            docker/docker-compose.middleware.yaml
-          services: |
-            sandbox
-            ssrf_proxy
-
-      - name: Run Workflow
-        run: dev/pytest/pytest_workflow.sh
-
-      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma)
-        uses: hoverkraft-tech/compose-action@v2.0.0
-        with:
-          compose-file: |
-            docker/docker-compose.middleware.yaml
-            docker/docker-compose.qdrant.yaml
-            docker/docker-compose.milvus.yaml
-            docker/docker-compose.pgvecto-rs.yaml
-            docker/docker-compose.pgvector.yaml
-            docker/docker-compose.chroma.yaml
-          services: |
-            weaviate
-            qdrant
-            etcd
-            minio
-            milvus-standalone
-            pgvecto-rs
-            pgvector
-            chroma
-
-      - name: Test Vector Stores
-        run: dev/pytest/pytest_vdb.sh
-
-  test-in-poetry:
     name: API Tests
     runs-on: ubuntu-latest
     strategy:
@@ -108,7 +40,7 @@ jobs:
 
       - name: Poetry check
         run: |
-          poetry check -C api
+          poetry check -C api --lock
           poetry show -C api
 
       - name: Install dependencies
@@ -123,6 +55,11 @@ jobs:
       - name: Run Tool
         run: poetry run -C api bash dev/pytest/pytest_tools.sh
 
+      - name: Set up dotenvs
+        run: |
+          cp docker/.env.example docker/.env
+          cp docker/middleware.env.example docker/middleware.env
+
       - name: Set up Sandbox
         uses: hoverkraft-tech/compose-action@v2.0.0
         with:
@@ -139,12 +76,7 @@ jobs:
         uses: hoverkraft-tech/compose-action@v2.0.0
         with:
           compose-file: |
-            docker/docker-compose.middleware.yaml
-            docker/docker-compose.qdrant.yaml
-            docker/docker-compose.milvus.yaml
-            docker/docker-compose.pgvecto-rs.yaml
-            docker/docker-compose.pgvector.yaml
-            docker/docker-compose.chroma.yaml
+            docker/docker-compose.yaml
           services: |
             weaviate
             qdrant
@@ -154,6 +86,5 @@ jobs:
             pgvecto-rs
             pgvector
             chroma
-
       - name: Test Vector Stores
         run: poetry run -C api bash dev/pytest/pytest_vdb.sh

.github/workflows/build-push.yml

@@ -8,6 +8,10 @@ on:
   release:
     types: [published]
 
+concurrency:
+  group: build-push-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 env:
   DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
   DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -15,19 +19,35 @@ env:
   DIFY_API_IMAGE_NAME: ${{ vars.DIFY_API_IMAGE_NAME || 'langgenius/dify-api' }}
 
 jobs:
-  build-and-push:
-    runs-on: ubuntu-latest
+  build:
+    runs-on: ${{ matrix.platform == 'linux/arm64' && 'arm64_runner' || 'ubuntu-latest' }}
     if: github.repository == 'langgenius/dify'
     strategy:
       matrix:
         include:
-          - service_name: "web"
-            image_name_env: "DIFY_WEB_IMAGE_NAME"
-            context: "web"
-          - service_name: "api"
+          - service_name: "build-api-amd64"
             image_name_env: "DIFY_API_IMAGE_NAME"
             context: "api"
+            platform: linux/amd64
+          - service_name: "build-api-arm64"
+            image_name_env: "DIFY_API_IMAGE_NAME"
+            context: "api"
+            platform: linux/arm64
+          - service_name: "build-web-amd64"
+            image_name_env: "DIFY_WEB_IMAGE_NAME"
+            context: "web"
+            platform: linux/amd64
+          - service_name: "build-web-arm64"
+            image_name_env: "DIFY_WEB_IMAGE_NAME"
+            context: "web"
+            platform: linux/arm64
+
     steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
@@ -40,7 +60,66 @@ jobs:
           username: ${{ env.DOCKERHUB_USER }}
           password: ${{ env.DOCKERHUB_TOKEN }}
 
-      - name: Extract metadata (tags, labels) for Docker
+      - name: Extract metadata for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env[matrix.image_name_env] }}
+
+      - name: Build Docker image
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: "{{defaultContext}}:${{ matrix.context }}"
+          platforms: ${{ matrix.platform }}
+          build-args: COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env[matrix.image_name_env] }},push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=gha,scope=${{ matrix.service_name }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.service_name }}
+
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.context }}-${{ env.PLATFORM_PAIR }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  create-manifest:
+    needs: build
+    runs-on: ubuntu-latest
+    if: github.repository == 'langgenius/dify'
+    strategy:
+      matrix:
+        include:
+          - service_name: "merge-api-images"
+            image_name_env: "DIFY_API_IMAGE_NAME"
+            context: "api"
+          - service_name: "merge-web-images"
+            image_name_env: "DIFY_WEB_IMAGE_NAME"
+            context: "web"
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-${{ matrix.context }}-*
+          merge-multiple: true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ env.DOCKERHUB_USER }}
+          password: ${{ env.DOCKERHUB_TOKEN }}
+
+      - name: Extract metadata for Docker
         id: meta
         uses: docker/metadata-action@v5
         with:
@@ -51,14 +130,12 @@ jobs:
             type=sha,enable=true,priority=100,prefix=,suffix=,format=long
             type=raw,value=${{ github.ref_name }},enable=${{ startsWith(github.ref, 'refs/tags/') }}
 
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: "{{defaultContext}}:${{ matrix.context }}"
-          platforms: ${{ startsWith(github.ref, 'refs/tags/') && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
-          build-args: COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env[matrix.image_name_env] }}@sha256:%s ' *)
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env[matrix.image_name_env] }}:${{ steps.meta.outputs.version }}

.github/workflows/db-migration-test.yml

@@ -38,13 +38,14 @@ jobs:
       - name: Install dependencies
         run: poetry install -C api
 
-      - name: Set up Middleware
+      - name: Set up Middlewares
         uses: hoverkraft-tech/compose-action@v2.0.0
         with:
           compose-file: |
             docker/docker-compose.middleware.yaml
           services: |
             db
+            redis
 
       - name: Prepare configs
         run: |
@@ -54,4 +55,4 @@ jobs:
       - name: Run DB Migration
         run: |
           cd api
-          poetry run python -m flask db upgrade
+          poetry run python -m flask upgrade-db

.github/workflows/style.yml

@@ -39,7 +39,7 @@ jobs:
 
       - name: Ruff check
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: poetry run -C api ruff check --preview ./api
+        run: poetry run -C api ruff check ./api
 
       - name: Dotenv check
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -99,7 +99,7 @@ jobs:
             **.sh
             **.yaml
             **.yml
-            Dockerfile
+            **Dockerfile
             dev/**
 
       - name: Super-linter
@@ -113,7 +113,8 @@ jobs:
           IGNORE_GITIGNORED_FILES: true
           VALIDATE_BASH: true
           VALIDATE_BASH_EXEC: true
-          VALIDATE_GITHUB_ACTIONS: true
+          # FIXME: temporarily disabled until api-docker.yaml's run script is fixed for shellcheck
+          # VALIDATE_GITHUB_ACTIONS: true
           VALIDATE_DOCKERFILE_HADOLINT: true
           VALIDATE_XML: true
           VALIDATE_YAML: true

.gitignore

@@ -136,11 +136,24 @@ web/.vscode/settings.json
 # Intellij IDEA Files
 .idea/*
 !.idea/vcs.xml
+!.idea/icon.png
 .ideaDataSources/
+*.iml
+api/.idea
 
 api/.env
 api/storage/*
 
+docker-legacy/volumes/app/storage/*
+docker-legacy/volumes/db/data/*
+docker-legacy/volumes/redis/data/*
+docker-legacy/volumes/weaviate/*
+docker-legacy/volumes/qdrant/*
+docker-legacy/volumes/etcd/*
+docker-legacy/volumes/minio/*
+docker-legacy/volumes/milvus/*
+docker-legacy/volumes/chroma/*
+
 docker/volumes/app/storage/*
 docker/volumes/db/data/*
 docker/volumes/redis/data/*
@@ -151,6 +164,9 @@ docker/volumes/minio/*
 docker/volumes/milvus/*
 docker/volumes/chroma/*
 
+docker/nginx/conf.d/default.conf
+docker/middleware.env
+
 sdks/python-client/build
 sdks/python-client/dist
 sdks/python-client/dify_client.egg-info

.vscode/launch.json

@@ -1,30 +1,16 @@
 {
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
-        {
-            "name": "Python: Celery",
-            "type": "debugpy",
-            "request": "launch",
-            "module": "celery",
-            "justMyCode": true,
-            "args": ["-A", "app.celery", "worker", "-P", "gevent", "-c", "1", "--loglevel", "info", "-Q", "dataset,generation,mail"],
-            "envFile": "${workspaceFolder}/.env",
-            "env": {
-                "FLASK_APP": "app.py",
-                "FLASK_DEBUG": "1",
-                "GEVENT_SUPPORT": "True"
-            },
-            "console": "integratedTerminal",
-            "python": "${command:python.interpreterPath}"
-        },
         {
             "name": "Python: Flask",
             "type": "debugpy",
             "request": "launch",
+            "python": "${workspaceFolder}/api/.venv/bin/python",
+            "cwd": "${workspaceFolder}/api",
+            "envFile": ".env",
             "module": "flask",
+            "justMyCode": true,
+            "jinja": true,
             "env": {
                 "FLASK_APP": "app.py",
                 "FLASK_DEBUG": "1",
@@ -34,11 +20,36 @@
                 "run",
                 "--host=0.0.0.0",
                 "--port=5001",
-                "--debug"
-            ],
-            "jinja": true,
+            ]
+        },
+        {
+            "name": "Python: Celery",
+            "type": "debugpy",
+            "request": "launch",
+            "python": "${workspaceFolder}/api/.venv/bin/python",
+            "cwd": "${workspaceFolder}/api",
+            "module": "celery",
             "justMyCode": true,
-            "python": "${command:python.interpreterPath}"
-        }
+            "envFile": ".env",
+            "console": "integratedTerminal",
+            "env": {
+                "FLASK_APP": "app.py",
+                "FLASK_DEBUG": "1",
+                "GEVENT_SUPPORT": "True"
+            },
+            "args": [
+                "-A",
+                "app.celery",
+                "worker",
+                "-P",
+                "gevent",
+                "-c",
+                "1",
+                "--loglevel",
+                "info",
+                "-Q",
+                "dataset,generation,mail,ops_trace"
+            ]
+        },
     ]
 }

README.md

@@ -174,6 +174,7 @@ The easiest way to start the Dify server is to run our [docker-compose.yml](dock
 
 ```bash
 cd docker
+cp .env.example .env
 docker compose up -d
 ```
 
@@ -183,7 +184,7 @@ After running, you can access the Dify dashboard in your browser at [http://loca
 
 ## Next steps
 
-If you need to customize the configuration, please refer to the comments in our [docker-compose.yml](docker/docker-compose.yaml) file and manually set the environment configuration. After making the changes, please run `docker-compose up -d` again. You can see the full list of environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
+If you need to customize the configuration, please refer to the comments in our [.env.example](docker/.env.example) file and update the corresponding values in your `.env` file. Additionally, you might need to make adjustments to the `docker-compose.yaml` file itself, such as changing image versions, port mappings, or volume mounts, based on your specific deployment environment and requirements. After making any changes, please re-run `docker-compose up -d`. You can find the full list of available environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
 If you'd like to configure a highly-available setup, there are community-contributed [Helm Charts](https://helm.sh/) and YAML files which allow Dify to be deployed on Kubernetes.
 

README_AR.md

@@ -157,15 +157,17 @@
 
 ```bash
 cd docker
+cp .env.example .env
 docker compose up -d
 ```
+
 بعد التشغيل، يمكنك الوصول إلى لوحة تحكم Dify في متصفحك على [http://localhost/install](http://localhost/install) وبدء عملية التهيئة.
 
 > إذا كنت ترغب في المساهمة في Dify أو القيام بتطوير إضافي، فانظر إلى [دليلنا للنشر من الشفرة (code) المصدرية](https://docs.dify.ai/getting-started/install-self-hosted/local-source-code)
 
 ## الخطوات التالية
 
-إذا كنت بحاجة إلى تخصيص التكوين، يرجى الرجوع إلى التعليقات في ملف [docker-compose.yml](docker/docker-compose.yaml) لدينا وتعيين التكوينات البيئية يدويًا. بعد إجراء التغييرات، يرجى تشغيل `docker-compose up -d` مرة أخرى. يمكنك رؤية قائمة كاملة بالمتغيرات البيئية [هنا](https://docs.dify.ai/getting-started/install-self-hosted/environments).
+إذا كنت بحاجة إلى تخصيص الإعدادات، فيرجى الرجوع إلى التعليقات في ملف [.env.example](docker/.env.example) وتحديث القيم المقابلة في ملف `.env`. بالإضافة إلى ذلك، قد تحتاج إلى إجراء تعديلات على ملف `docker-compose.yaml` نفسه، مثل تغيير إصدارات الصور أو تعيينات المنافذ أو نقاط تحميل وحدات التخزين، بناءً على بيئة النشر ومتطلباتك الخاصة. بعد إجراء أي تغييرات، يرجى إعادة تشغيل `docker-compose up -d`. يمكنك العثور على قائمة كاملة بمتغيرات البيئة المتاحة [هنا](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
 يوجد مجتمع خاص بـ [Helm Charts](https://helm.sh/) وملفات YAML التي تسمح بتنفيذ Dify على Kubernetes للنظام من الإيجابيات العلوية.
 

README_CN.md

@@ -179,11 +179,16 @@ Dify 是一个开源的 LLM 应用开发平台。其直观的界面结合了 AI
 
 ```bash
 cd docker
+cp .env.example .env
 docker compose up -d
 ```
 
 运行后，可以在浏览器上访问 [http://localhost/install](http://localhost/install) 进入 Dify 控制台并开始初始化安装操作。
 
+### 自定义配置
+
+如果您需要自定义配置，请参考 [.env.example](docker/.env.example) 文件中的注释，并更新 `.env` 文件中对应的值。此外，您可能需要根据您的具体部署环境和需求对 `docker-compose.yaml` 文件本身进行调整，例如更改镜像版本、端口映射或卷挂载。完成任何更改后，请重新运行 `docker-compose up -d`。您可以在[此处](https://docs.dify.ai/getting-started/install-self-hosted/environments)找到可用环境变量的完整列表。
+
 #### 使用 Helm Chart 部署
 
 使用 [Helm Chart](https://helm.sh/) 版本或者 YAML 文件，可以在 Kubernetes 上部署 Dify。
@@ -192,10 +197,6 @@ docker compose up -d
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [YAML 文件 by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
-### 配置
-
-如果您需要自定义配置，请参考我们的 [docker-compose.yml](docker/docker-compose.yaml) 文件中的注释，并手动设置环境配置。更改后，请再次运行 `docker-compose up -d`。您可以在我们的[文档](https://docs.dify.ai/getting-started/install-self-hosted/environments)中查看所有环境变量的完整列表。
-
 ## Star History
 
 [![Star History Chart](https://api.star-history.com/svg?repos=langgenius/dify&type=Date)](https://star-history.com/#langgenius/dify&Date)

README_ES.md

@@ -179,6 +179,7 @@ La forma más fácil de iniciar el servidor de Dify es ejecutar nuestro archivo
 
 ```bash
 cd docker
+cp .env.example .env
 docker compose up -d
 ```
 
@@ -188,7 +189,7 @@ Después de ejecutarlo, puedes acceder al panel de control de Dify en tu navegad
 
 ## Próximos pasos
 
-Si necesitas personalizar la configuración, consulta los comentarios en nuestro archivo [docker-compose.yml](docker/docker-compose.yaml) y configura manualmente la configuración del entorno
+Si necesita personalizar la configuración, consulte los comentarios en nuestro archivo [.env.example](docker/.env.example) y actualice los valores correspondientes en su archivo `.env`. Además, es posible que deba realizar ajustes en el propio archivo `docker-compose.yaml`, como cambiar las versiones de las imágenes, las asignaciones de puertos o los montajes de volúmenes, según su entorno de implementación y requisitos específicos. Después de realizar cualquier cambio, vuelva a ejecutar `docker-compose up -d`. Puede encontrar la lista completa de variables de entorno disponibles [aquí](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
 . Después de realizar los cambios, ejecuta `docker-compose up -d` nuevamente. Puedes ver la lista completa de variables de entorno [aquí](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 

README_FR.md

@@ -179,6 +179,7 @@ La manière la plus simple de démarrer le serveur Dify est d'exécuter notre fi
 
 ```bash
 cd docker
+cp .env.example .env
 docker compose up -d
 ```
 
@@ -188,9 +189,7 @@ Après l'exécution, vous pouvez accéder au tableau de bord Dify dans votre nav
 
 ## Prochaines étapes
 
-Si vous devez personnaliser la configuration, veuillez
-
- vous référer aux commentaires dans notre fichier [docker-compose.yml](docker/docker-compose.yaml) et définir manuellement la configuration de l'environnement. Après avoir apporté les modifications, veuillez exécuter à nouveau `docker-compose up -d`. Vous pouvez voir la liste complète des variables d'environnement [ici](https://docs.dify.ai/getting-started/install-self-hosted/environments).
+Si vous devez personnaliser la configuration, veuillez vous référer aux commentaires dans notre fichier [.env.example](docker/.env.example) et mettre à jour les valeurs correspondantes dans votre fichier `.env`. De plus, vous devrez peut-être apporter des modifications au fichier `docker-compose.yaml` lui-même, comme changer les versions d'image, les mappages de ports ou les montages de volumes, en fonction de votre environnement de déploiement et de vos exigences spécifiques. Après avoir effectué des modifications, veuillez réexécuter `docker-compose up -d`. Vous pouvez trouver la liste complète des variables d'environnement disponibles [ici](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
 Si vous souhaitez configurer une configuration haute disponibilité, la communauté fournit des [Helm Charts](https://helm.sh/) et des fichiers YAML, à travers lesquels vous pouvez déployer Dify sur Kubernetes.
 

README_JA.md

@@ -178,6 +178,7 @@ Difyサーバーを起動する最も簡単な方法は、[docker-compose.yml](d
 
 ```bash
 cd docker
+cp .env.example .env
 docker compose up -d
 ```
 
@@ -187,7 +188,7 @@ docker compose up -d
 
 ## 次のステップ
 
-環境設定をカスタマイズする場合は、[docker-compose.yml](docker/docker-compose.yaml)ファイル内のコメントを参照して、環境設定を手動で設定してください。変更を加えた後は、再び `docker-compose up -d` を実行してください。環境変数の完全なリストは[こちら](https://docs.dify.ai/getting-started/install-self-hosted/environments)をご覧ください。
+設定をカスタマイズする必要がある場合は、[.env.example](docker/.env.example) ファイルのコメントを参照し、`.env` ファイルの対応する値を更新してください。さらに、デプロイ環境や要件に応じて、`docker-compose.yaml` ファイル自体を調整する必要がある場合があります。たとえば、イメージのバージョン、ポートのマッピング、ボリュームのマウントなどを変更します。変更を加えた後は、`docker-compose up -d` を再実行してください。利用可能な環境変数の全一覧は、[こちら](https://docs.dify.ai/getting-started/install-self-hosted/environments)で確認できます。
 
 高可用性設定を設定する必要がある場合、コミュニティは[Helm Charts](https://helm.sh/)とYAMLファイルにより、DifyをKubernetesにデプロイすることができます。
 

README_KL.md

@@ -179,6 +179,7 @@ The easiest way to start the Dify server is to run our [docker-compose.yml](dock
 
 ```bash
 cd docker
+cp .env.example .env
 docker compose up -d
 ```
 
@@ -188,7 +189,7 @@ After running, you can access the Dify dashboard in your browser at [http://loca
 
 ## Next steps
 
-If you need to customize the configuration, please refer to the comments in our [docker-compose.yml](docker/docker-compose.yaml) file and manually set the environment configuration. After making the changes, please run `docker-compose up -d` again. You can see the full list of environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
+If you need to customize the configuration, please refer to the comments in our [.env.example](docker/.env.example) file and update the corresponding values in your `.env` file. Additionally, you might need to make adjustments to the `docker-compose.yaml` file itself, such as changing image versions, port mappings, or volume mounts, based on your specific deployment environment and requirements. After making any changes, please re-run `docker-compose up -d`. You can find the full list of available environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
 If you'd like to configure a highly-available setup, there are community-contributed [Helm Charts](https://helm.sh/) and YAML files which allow Dify to be deployed on Kubernetes.
 

README_KR.md

@@ -172,6 +172,7 @@ Dify 서버를 시작하는 가장 쉬운 방법은 [docker-compose.yml](docker/
 
 ```bash
 cd docker
+cp .env.example .env
 docker compose up -d
 ```
 
@@ -181,8 +182,7 @@ docker compose up -d
 
 ## 다음 단계
 
-구성 커스터마이징이 필요한 경우, [docker-compose.yml](docker/docker-compose.yaml) 파일의 코멘트를 참조하여 환경 구성을 수동으로 설정하십시오. 변경 후 `docker-compose up -d` 를 다시 실행하십시오. 환경 변수의 전체 목록은 [여기](https://docs.dify.ai/getting-started/install-self-hosted/environments)에서 확인할 수 있습니다.
-
+구성을 사용자 정의해야 하는 경우 [.env.example](docker/.env.example) 파일의 주석을 참조하고 `.env` 파일에서 해당 값을 업데이트하십시오. 또한 특정 배포 환경 및 요구 사항에 따라 `docker-compose.yaml` 파일 자체를 조정해야 할 수도 있습니다. 예를 들어 이미지 버전, 포트 매핑 또는 볼륨 마운트를 변경합니다. 변경 한 후 `docker-compose up -d`를 다시 실행하십시오. 사용 가능한 환경 변수의 전체 목록은 [여기](https://docs.dify.ai/getting-started/install-self-hosted/environments)에서 찾을 수 있습니다.
 
 Dify를 Kubernetes에 배포하고 프리미엄 스케일링 설정을 구성했다는 커뮤니티가 제공하는 [Helm Charts](https://helm.sh/)와 YAML 파일이 존재합니다.
 

api/.dockerignore

@@ -8,4 +8,7 @@ logs
 *.log*
 
 # jetbrains
-.idea
+.idea
+
+# venv
+.venv

api/.env.example

@@ -39,7 +39,7 @@ DB_DATABASE=dify
 
 # Storage configuration
 # use for store upload files, private keys...
-# storage type: local, s3, azure-blob
+# storage type: local, s3, azure-blob, google-storage
 STORAGE_TYPE=local
 STORAGE_LOCAL_PATH=storage
 S3_USE_AWS_MANAGED_IAM=false
@@ -63,13 +63,20 @@ ALIYUN_OSS_REGION=your-region
 
 # Google Storage configuration
 GOOGLE_STORAGE_BUCKET_NAME=yout-bucket-name
-GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON=your-google-service-account-json-base64-string
+GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64=your-google-service-account-json-base64-string
+
+# Tencent COS Storage configuration
+TENCENT_COS_BUCKET_NAME=your-bucket-name
+TENCENT_COS_SECRET_KEY=your-secret-key
+TENCENT_COS_SECRET_ID=your-secret-id
+TENCENT_COS_REGION=your-region
+TENCENT_COS_SCHEME=your-scheme
 
 # CORS configuration
 WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 
-# Vector database configuration, support: weaviate, qdrant, milvus, relyt, pgvecto_rs, pgvector
+# Vector database configuration, support: weaviate, qdrant, milvus, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector
 VECTOR_STORE=weaviate
 
 # Weaviate configuration
@@ -137,6 +144,13 @@ CHROMA_DATABASE=default_database
 CHROMA_AUTH_PROVIDER=chromadb.auth.token_authn.TokenAuthenticationServerProvider
 CHROMA_AUTH_CREDENTIALS=difyai123456
 
+# OpenSearch configuration
+OPENSEARCH_HOST=127.0.0.1
+OPENSEARCH_PORT=9200
+OPENSEARCH_USER=admin
+OPENSEARCH_PASSWORD=admin
+OPENSEARCH_SECURE=true
+
 # Upload configuration
 UPLOAD_FILE_SIZE_LIMIT=15
 UPLOAD_FILE_BATCH_LIMIT=5

api/Dockerfile

@@ -1,34 +1,43 @@
 # base image
 FROM python:3.10-slim-bookworm AS base
 
-LABEL maintainer="takatost@gmail.com"
+WORKDIR /app/api
 
-# install packages
-FROM base as packages
+# Install Poetry
+ENV POETRY_VERSION=1.8.3
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir --upgrade poetry==${POETRY_VERSION}
+
+# Configure Poetry
+ENV POETRY_CACHE_DIR=/tmp/poetry_cache
+ENV POETRY_NO_INTERACTION=1
+ENV POETRY_VIRTUALENVS_IN_PROJECT=true
+ENV POETRY_VIRTUALENVS_CREATE=true
+
+FROM base AS packages
 
 RUN apt-get update \
     && apt-get install -y --no-install-recommends gcc g++ libc-dev libffi-dev libgmp-dev libmpfr-dev libmpc-dev
 
-COPY requirements.txt /requirements.txt
-
-RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install --prefix=/pkg -r requirements.txt
+# Install Python dependencies
+COPY pyproject.toml poetry.lock ./
+RUN poetry install --sync --no-cache --no-root
 
 # production stage
 FROM base AS production
 
-ENV FLASK_APP app.py
-ENV EDITION SELF_HOSTED
-ENV DEPLOY_ENV PRODUCTION
-ENV CONSOLE_API_URL http://127.0.0.1:5001
-ENV CONSOLE_WEB_URL http://127.0.0.1:3000
-ENV SERVICE_API_URL http://127.0.0.1:5001
-ENV APP_WEB_URL http://127.0.0.1:3000
+ENV FLASK_APP=app.py
+ENV EDITION=SELF_HOSTED
+ENV DEPLOY_ENV=PRODUCTION
+ENV CONSOLE_API_URL=http://127.0.0.1:5001
+ENV CONSOLE_WEB_URL=http://127.0.0.1:3000
+ENV SERVICE_API_URL=http://127.0.0.1:5001
+ENV APP_WEB_URL=http://127.0.0.1:3000
 
 EXPOSE 5001
 
 # set timezone
-ENV TZ UTC
+ENV TZ=UTC
 
 WORKDIR /app/api
 
@@ -37,13 +46,20 @@ RUN apt-get update \
     && apt-get autoremove \
     && rm -rf /var/lib/apt/lists/*
 
-COPY --from=packages /pkg /usr/local
+# Copy Python environment and packages
+ENV VIRTUAL_ENV=/app/api/.venv
+COPY --from=packages ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
+
+# Copy source code
 COPY . /app/api/
 
+# Copy entrypoint
 COPY docker/entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 
-ARG COMMIT_SHA
-ENV COMMIT_SHA ${COMMIT_SHA}
 
-ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
+ARG COMMIT_SHA
+ENV COMMIT_SHA=${COMMIT_SHA}
+
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]

api/README.md

@@ -2,33 +2,43 @@
 
 ## Usage
 
+> [!IMPORTANT]
+> In the v0.6.12 release, we deprecated `pip` as the package management tool for Dify API Backend service and replaced it with `poetry`.
+
 1. Start the docker-compose stack
 
    The backend require some middleware, including PostgreSQL, Redis, and Weaviate, which can be started together using `docker-compose`.
 
    ```bash
    cd ../docker
-   docker-compose -f docker-compose.middleware.yaml -p dify up -d
+   docker compose -f docker-compose.middleware.yaml -p dify up -d
    cd ../api
    ```
+
 2. Copy `.env.example` to `.env`
 3. Generate a `SECRET_KEY` in the `.env` file.
 
-   ```bash
+   ```bash for Linux
    sed -i "/^SECRET_KEY=/c\SECRET_KEY=$(openssl rand -base64 42)" .env
    ```
+
+   ```bash for Mac
+   secret_key=$(openssl rand -base64 42)
+   sed -i '' "/^SECRET_KEY=/c\\
+   SECRET_KEY=${secret_key}" .env
+   ```
+
 4. Create environment.
 
    Dify API service uses [Poetry](https://python-poetry.org/docs/) to manage dependencies. You can execute `poetry shell` to activate the environment.
 
-   > Using pip can be found [below](#usage-with-pip). 
-   
-6. Install dependencies
-   
+5. Install dependencies
+
    ```bash
+   poetry env use 3.10
    poetry install
    ```
-   
+
    In case of contributors missing to update dependencies for `pyproject.toml`, you can perform the following shell instead.
 
    ```bash
@@ -37,113 +47,41 @@
    poetry add $(cat requirements-dev.txt) --group dev    # install dependencies of development and update pyproject.toml
    ```
 
-7. Run migrate
+6. Run migrate
 
    Before the first launch, migrate the database to the latest version.
-   
+
    ```bash
    poetry run python -m flask db upgrade
    ```
 
-8. Start backend
-    
+7. Start backend
+
    ```bash
    poetry run python -m flask run --host 0.0.0.0 --port=5001 --debug
    ```
 
-9. Start Dify [web](../web) service.
-10. Setup your application by visiting `http://localhost:3000`...
-11. If you need to debug local async processing, please start the worker service.
+8. Start Dify [web](../web) service.
+9. Setup your application by visiting `http://localhost:3000`...
+10. If you need to debug local async processing, please start the worker service.
 
    ```bash
-   poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail
+   poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace
    ```
 
    The started celery app handles the async tasks, e.g. dataset importing and documents indexing.
 
-
 ## Testing
 
 1. Install dependencies for both the backend and the test environment
-   
+
    ```bash
    poetry install --with dev
-   ``` 
-   
+   ```
+
 2. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml`
-   
+
    ```bash
    cd ../
    poetry run -C api bash dev/pytest/pytest_all_tests.sh
    ```
-
-
-## Usage with pip
-
-> [!NOTE]  
-> In the next version, we will deprecate pip as the primary package management tool for dify api service, currently Poetry and pip coexist.
-
-1. Start the docker-compose stack
-
-   The backend require some middleware, including PostgreSQL, Redis, and Weaviate, which can be started together using `docker-compose`.
-
-   ```bash
-   cd ../docker
-   docker-compose -f docker-compose.middleware.yaml -p dify up -d
-   cd ../api
-   ```
-   
-2. Copy `.env.example` to `.env`
-3. Generate a `SECRET_KEY` in the `.env` file.
-
-   ```bash
-   sed -i "/^SECRET_KEY=/c\SECRET_KEY=$(openssl rand -base64 42)" .env
-   ```
-
-4. Create environment.
-   
-   If you use Anaconda, create a new environment and activate it
-  
-   ```bash
-   conda create --name dify python=3.10
-   conda activate dify
-   ```
-      
-6. Install dependencies
-   
-   ```bash
-   pip install -r requirements.txt
-   ```
-
-7. Run migrate
-
-   Before the first launch, migrate the database to the latest version.
-
-   ```bash
-   flask db upgrade
-   ```
-
-8. Start backend:
-   ```bash
-   flask run --host 0.0.0.0 --port=5001 --debug
-   ```
-9. Setup your application by visiting http://localhost:5001/console/api/setup or other apis...
-10. If you need to debug local async processing, please start the worker service.
-   ```bash
-   celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail
-   ```
-   The started celery app handles the async tasks, e.g. dataset importing and documents indexing.
-
-
-## Testing
-
-1. Install dependencies for both the backend and the test environment
-   ```bash
-   pip install -r requirements.txt -r requirements-dev.txt
-   ``` 
-   
-2. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml`
-   ```bash
-   dev/pytest/pytest_all_tests.sh
-   ```
-

api/app.py

@@ -1,6 +1,8 @@
 import os
 
-if not os.environ.get("DEBUG") or os.environ.get("DEBUG").lower() != 'true':
+from configs.app_config import DifyConfig
+
+if not os.environ.get("DEBUG") or os.environ.get("DEBUG", "false").lower() != 'true':
     from gevent import monkey
 
     monkey.patch_all()
@@ -22,7 +24,6 @@ from flask_cors import CORS
 from werkzeug.exceptions import Unauthorized
 
 from commands import register_commands
-from config import Config
 
 # DO NOT REMOVE BELOW
 from events import event_handlers
@@ -74,10 +75,28 @@ config_type = os.getenv('EDITION', default='SELF_HOSTED')  # ce edition first
 # Application Factory Function
 # ----------------------------
 
+def create_flask_app_with_configs() -> Flask:
+    """
+    create a raw flask app
+    with configs loaded from .env file
+    """
+    dify_app = DifyApp(__name__)
+    dify_app.config.from_mapping(DifyConfig().model_dump())
+
+    # populate configs into system environment variables
+    for key, value in dify_app.config.items():
+        if isinstance(value, str):
+            os.environ[key] = value
+        elif isinstance(value, int | float | bool):
+            os.environ[key] = str(value)
+        elif value is None:
+            os.environ[key] = ''
+
+    return dify_app
+
 
 def create_app() -> Flask:
-    app = DifyApp(__name__)
-    app.config.from_object(Config())
+    app = create_flask_app_with_configs()
 
     app.secret_key = app.config['SECRET_KEY']
 
@@ -99,9 +118,21 @@ def create_app() -> Flask:
         level=app.config.get('LOG_LEVEL'),
         format=app.config.get('LOG_FORMAT'),
         datefmt=app.config.get('LOG_DATEFORMAT'),
-        handlers=log_handlers
+        handlers=log_handlers,
+        force=True
     )
+    log_tz = app.config.get('LOG_TZ')
+    if log_tz:
+        from datetime import datetime
 
+        import pytz
+        timezone = pytz.timezone(log_tz)
+
+        def time_converter(seconds):
+            return datetime.utcfromtimestamp(seconds).astimezone(timezone).timetuple()
+
+        for handler in logging.root.handlers:
+            handler.formatter.converter = time_converter
     initialize_extensions(app)
     register_blueprints(app)
     register_commands(app)
@@ -129,27 +160,26 @@ def initialize_extensions(app):
 @login_manager.request_loader
 def load_user_from_request(request_from_flask_login):
     """Load user based on the request."""
-    if request.blueprint in ['console', 'inner_api']:
-        # Check if the user_id contains a dot, indicating the old format
-        auth_header = request.headers.get('Authorization', '')
-        if not auth_header:
-            auth_token = request.args.get('_token')
-            if not auth_token:
-                raise Unauthorized('Invalid Authorization token.')
-        else:
-            if ' ' not in auth_header:
-                raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.')
-            auth_scheme, auth_token = auth_header.split(None, 1)
-            auth_scheme = auth_scheme.lower()
-            if auth_scheme != 'bearer':
-                raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.')
-
-        decoded = PassportService().verify(auth_token)
-        user_id = decoded.get('user_id')
-
-        return AccountService.load_user(user_id)
-    else:
+    if request.blueprint not in ['console', 'inner_api']:
         return None
+    # Check if the user_id contains a dot, indicating the old format
+    auth_header = request.headers.get('Authorization', '')
+    if not auth_header:
+        auth_token = request.args.get('_token')
+        if not auth_token:
+            raise Unauthorized('Invalid Authorization token.')
+    else:
+        if ' ' not in auth_header:
+            raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.')
+        auth_scheme, auth_token = auth_header.split(None, 1)
+        auth_scheme = auth_scheme.lower()
+        if auth_scheme != 'bearer':
+            raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.')
+
+    decoded = PassportService().verify(auth_token)
+    user_id = decoded.get('user_id')
+
+    return AccountService.load_logged_in_account(account_id=user_id, token=auth_token)
 
 
 @login_manager.unauthorized_handler
@@ -210,7 +240,7 @@ def register_blueprints(app):
 app = create_app()
 celery = app.extensions["celery"]
 
-if app.config['TESTING']:
+if app.config.get('TESTING'):
     print("App is running in TESTING mode")
 
 

api/commands.py

@@ -1,5 +1,6 @@
 import base64
 import json
+import logging
 import secrets
 from typing import Optional
 
@@ -11,7 +12,9 @@ from constants.languages import languages
 from core.rag.datasource.vdb.vector_factory import Vector
 from core.rag.datasource.vdb.vector_type import VectorType
 from core.rag.models.document import Document
+from events.app_event import app_was_created
 from extensions.ext_database import db
+from extensions.ext_redis import redis_client
 from libs.helper import email as email_validate
 from libs.password import hash_password, password_pattern, valid_password
 from libs.rsa import generate_key_pair
@@ -325,6 +328,14 @@ def migrate_knowledge_vector_database():
                         "vector_store": {"class_prefix": collection_name}
                     }
                     dataset.index_struct = json.dumps(index_struct_dict)
+                elif vector_type == VectorType.OPENSEARCH:
+                    dataset_id = dataset.id
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+                    index_struct_dict = {
+                        "type": VectorType.OPENSEARCH,
+                        "vector_store": {"class_prefix": collection_name}
+                    }
+                    dataset.index_struct = json.dumps(index_struct_dict)
                 else:
                     raise ValueError(f"Vector store {vector_type} is not supported.")
 
@@ -553,6 +564,75 @@ def create_tenant(email: str, language: Optional[str] = None):
                            'Account: {}\nPassword: {}'.format(email, new_password), fg='green'))
 
 
+@click.command('upgrade-db', help='upgrade the database')
+def upgrade_db():
+    click.echo('Preparing database migration...')
+    lock = redis_client.lock(name='db_upgrade_lock', timeout=60)
+    if lock.acquire(blocking=False):
+        try:
+            click.echo(click.style('Start database migration.', fg='green'))
+
+            # run db migration
+            import flask_migrate
+            flask_migrate.upgrade()
+
+            click.echo(click.style('Database migration successful!', fg='green'))
+
+        except Exception as e:
+            logging.exception(f'Database migration failed, error: {e}')
+        finally:
+            lock.release()
+    else:
+        click.echo('Database migration skipped')
+
+
+@click.command('fix-app-site-missing', help='Fix app related site missing issue.')
+def fix_app_site_missing():
+    """
+    Fix app related site missing issue.
+    """
+    click.echo(click.style('Start fix app related site missing issue.', fg='green'))
+
+    failed_app_ids = []
+    while True:
+        sql = """select apps.id as id from apps left join sites on sites.app_id=apps.id
+where sites.id is null limit 1000"""
+        with db.engine.begin() as conn:
+            rs = conn.execute(db.text(sql))
+
+            processed_count = 0
+            for i in rs:
+                processed_count += 1
+                app_id = str(i.id)
+
+                if app_id in failed_app_ids:
+                    continue
+
+                try:
+                    app = db.session.query(App).filter(App.id == app_id).first()
+                    tenant = app.tenant
+                    if tenant:
+                        accounts = tenant.get_accounts()
+                        if not accounts:
+                            print("Fix app {} failed.".format(app.id))
+                            continue
+
+                        account = accounts[0]
+                        print("Fix app {} related site missing issue.".format(app.id))
+                        app_was_created.send(app, account=account)
+                except Exception as e:
+                    failed_app_ids.append(app_id)
+                    click.echo(click.style('Fix app {} related site missing issue failed!'.format(app_id), fg='red'))
+                    logging.exception(f'Fix app related site missing issue failed, error: {e}')
+                    continue
+
+            if not processed_count:
+                break
+
+
+    click.echo(click.style('Congratulations! Fix app related site missing issue successful!', fg='green'))
+
+
 def register_commands(app):
     app.cli.add_command(reset_password)
     app.cli.add_command(reset_email)
@@ -561,4 +641,5 @@ def register_commands(app):
     app.cli.add_command(convert_to_agent_apps)
     app.cli.add_command(add_qdrant_doc_id_index)
     app.cli.add_command(create_tenant)
-
+    app.cli.add_command(upgrade_db)
+    app.cli.add_command(fix_app_site_missing)

api/config.py

@@ -1,451 +0,0 @@
-import os
-
-import dotenv
-
-dotenv.load_dotenv()
-
-DEFAULTS = {
-    'EDITION': 'SELF_HOSTED',
-    'DB_USERNAME': 'postgres',
-    'DB_PASSWORD': '',
-    'DB_HOST': 'localhost',
-    'DB_PORT': '5432',
-    'DB_DATABASE': 'dify',
-    'DB_CHARSET': '',
-    'REDIS_HOST': 'localhost',
-    'REDIS_PORT': '6379',
-    'REDIS_DB': '0',
-    'REDIS_USE_SSL': 'False',
-    'OAUTH_REDIRECT_PATH': '/console/api/oauth/authorize',
-    'OAUTH_REDIRECT_INDEX_PATH': '/',
-    'CONSOLE_WEB_URL': 'https://cloud.dify.ai',
-    'CONSOLE_API_URL': 'https://cloud.dify.ai',
-    'SERVICE_API_URL': 'https://api.dify.ai',
-    'APP_WEB_URL': 'https://udify.app',
-    'FILES_URL': '',
-    'FILES_ACCESS_TIMEOUT': 300,
-    'S3_USE_AWS_MANAGED_IAM': 'False',
-    'S3_ADDRESS_STYLE': 'auto',
-    'STORAGE_TYPE': 'local',
-    'STORAGE_LOCAL_PATH': 'storage',
-    'CHECK_UPDATE_URL': 'https://updates.dify.ai',
-    'DEPLOY_ENV': 'PRODUCTION',
-    'SQLALCHEMY_DATABASE_URI_SCHEME': 'postgresql',
-    'SQLALCHEMY_POOL_SIZE': 30,
-    'SQLALCHEMY_MAX_OVERFLOW': 10,
-    'SQLALCHEMY_POOL_RECYCLE': 3600,
-    'SQLALCHEMY_POOL_PRE_PING': 'False',
-    'SQLALCHEMY_ECHO': 'False',
-    'SENTRY_TRACES_SAMPLE_RATE': 1.0,
-    'SENTRY_PROFILES_SAMPLE_RATE': 1.0,
-    'WEAVIATE_GRPC_ENABLED': 'True',
-    'WEAVIATE_BATCH_SIZE': 100,
-    'QDRANT_CLIENT_TIMEOUT': 20,
-    'QDRANT_GRPC_ENABLED': 'False',
-    'QDRANT_GRPC_PORT': '6334',
-    'CELERY_BACKEND': 'database',
-    'LOG_LEVEL': 'INFO',
-    'LOG_FILE': '',
-    'LOG_FORMAT': '%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s',
-    'LOG_DATEFORMAT': '%Y-%m-%d %H:%M:%S',
-    'HOSTED_OPENAI_QUOTA_LIMIT': 200,
-    'HOSTED_OPENAI_TRIAL_ENABLED': 'False',
-    'HOSTED_OPENAI_TRIAL_MODELS': 'gpt-3.5-turbo,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-16k,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-0613,gpt-3.5-turbo-0125,text-davinci-003',
-    'HOSTED_OPENAI_PAID_ENABLED': 'False',
-    'HOSTED_OPENAI_PAID_MODELS': 'gpt-4,gpt-4-turbo-preview,gpt-4-turbo-2024-04-09,gpt-4-1106-preview,gpt-4-0125-preview,gpt-3.5-turbo,gpt-3.5-turbo-16k,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-1106,gpt-3.5-turbo-0613,gpt-3.5-turbo-0125,gpt-3.5-turbo-instruct,text-davinci-003',
-    'HOSTED_AZURE_OPENAI_ENABLED': 'False',
-    'HOSTED_AZURE_OPENAI_QUOTA_LIMIT': 200,
-    'HOSTED_ANTHROPIC_QUOTA_LIMIT': 600000,
-    'HOSTED_ANTHROPIC_TRIAL_ENABLED': 'False',
-    'HOSTED_ANTHROPIC_PAID_ENABLED': 'False',
-    'HOSTED_MODERATION_ENABLED': 'False',
-    'HOSTED_MODERATION_PROVIDERS': '',
-    'HOSTED_FETCH_APP_TEMPLATES_MODE': 'remote',
-    'HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN': 'https://tmpl.dify.ai',
-    'CLEAN_DAY_SETTING': 30,
-    'UPLOAD_FILE_SIZE_LIMIT': 15,
-    'UPLOAD_FILE_BATCH_LIMIT': 5,
-    'UPLOAD_IMAGE_FILE_SIZE_LIMIT': 10,
-    'OUTPUT_MODERATION_BUFFER_SIZE': 300,
-    'MULTIMODAL_SEND_IMAGE_FORMAT': 'base64',
-    'INVITE_EXPIRY_HOURS': 72,
-    'BILLING_ENABLED': 'False',
-    'CAN_REPLACE_LOGO': 'False',
-    'MODEL_LB_ENABLED': 'False',
-    'ETL_TYPE': 'dify',
-    'KEYWORD_STORE': 'jieba',
-    'BATCH_UPLOAD_LIMIT': 20,
-    'CODE_EXECUTION_ENDPOINT': 'http://sandbox:8194',
-    'CODE_EXECUTION_API_KEY': 'dify-sandbox',
-    'TOOL_ICON_CACHE_MAX_AGE': 3600,
-    'MILVUS_DATABASE': 'default',
-    'KEYWORD_DATA_SOURCE_TYPE': 'database',
-    'INNER_API': 'False',
-    'ENTERPRISE_ENABLED': 'False',
-    'INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH': 1000,
-    'WORKFLOW_MAX_EXECUTION_STEPS': 500,
-    'WORKFLOW_MAX_EXECUTION_TIME': 1200,
-    'WORKFLOW_CALL_MAX_DEPTH': 5,
-    'APP_MAX_EXECUTION_TIME': 1200,
-}
-
-
-def get_env(key):
-    return os.environ.get(key, DEFAULTS.get(key))
-
-
-def get_bool_env(key):
-    value = get_env(key)
-    return value.lower() == 'true' if value is not None else False
-
-
-def get_cors_allow_origins(env, default):
-    cors_allow_origins = []
-    if get_env(env):
-        for origin in get_env(env).split(','):
-            cors_allow_origins.append(origin)
-    else:
-        cors_allow_origins = [default]
-
-    return cors_allow_origins
-
-
-class Config:
-    """Application configuration class."""
-
-    def __init__(self):
-        # ------------------------
-        # General Configurations.
-        # ------------------------
-        self.CURRENT_VERSION = "0.6.11"
-        self.COMMIT_SHA = get_env('COMMIT_SHA')
-        self.EDITION = get_env('EDITION')
-        self.DEPLOY_ENV = get_env('DEPLOY_ENV')
-        self.TESTING = False
-        self.LOG_LEVEL = get_env('LOG_LEVEL')
-        self.LOG_FILE = get_env('LOG_FILE')
-        self.LOG_FORMAT = get_env('LOG_FORMAT')
-        self.LOG_DATEFORMAT = get_env('LOG_DATEFORMAT')
-        self.API_COMPRESSION_ENABLED = get_bool_env('API_COMPRESSION_ENABLED')
-
-        # The backend URL prefix of the console API.
-        # used to concatenate the login authorization callback or notion integration callback.
-        self.CONSOLE_API_URL = get_env('CONSOLE_API_URL')
-
-        # The front-end URL prefix of the console web.
-        # used to concatenate some front-end addresses and for CORS configuration use.
-        self.CONSOLE_WEB_URL = get_env('CONSOLE_WEB_URL')
-
-        # WebApp Url prefix.
-        # used to display WebAPP API Base Url to the front-end.
-        self.APP_WEB_URL = get_env('APP_WEB_URL')
-
-        # Service API Url prefix.
-        # used to display Service API Base Url to the front-end.
-        self.SERVICE_API_URL = get_env('SERVICE_API_URL')
-
-        # File preview or download Url prefix.
-        # used to display File preview or download Url to the front-end or as Multi-model inputs;
-        # Url is signed and has expiration time.
-        self.FILES_URL = get_env('FILES_URL') if get_env('FILES_URL') else self.CONSOLE_API_URL
-
-        # File Access Time specifies a time interval in seconds for the file to be accessed.
-        # The default value is 300 seconds.
-        self.FILES_ACCESS_TIMEOUT = int(get_env('FILES_ACCESS_TIMEOUT'))
-
-        # Your App secret key will be used for securely signing the session cookie
-        # Make sure you are changing this key for your deployment with a strong key.
-        # You can generate a strong key using `openssl rand -base64 42`.
-        # Alternatively you can set it with `SECRET_KEY` environment variable.
-        self.SECRET_KEY = get_env('SECRET_KEY')
-
-        # Enable or disable the inner API.
-        self.INNER_API = get_bool_env('INNER_API')
-        # The inner API key is used to authenticate the inner API.
-        self.INNER_API_KEY = get_env('INNER_API_KEY')
-
-        # cors settings
-        self.CONSOLE_CORS_ALLOW_ORIGINS = get_cors_allow_origins(
-            'CONSOLE_CORS_ALLOW_ORIGINS', self.CONSOLE_WEB_URL)
-        self.WEB_API_CORS_ALLOW_ORIGINS = get_cors_allow_origins(
-            'WEB_API_CORS_ALLOW_ORIGINS', '*')
-
-        # check update url
-        self.CHECK_UPDATE_URL = get_env('CHECK_UPDATE_URL')
-
-        # ------------------------
-        # Database Configurations.
-        # ------------------------
-        db_credentials = {
-            key: get_env(key) for key in
-            ['DB_USERNAME', 'DB_PASSWORD', 'DB_HOST', 'DB_PORT', 'DB_DATABASE', 'DB_CHARSET']
-        }
-        self.SQLALCHEMY_DATABASE_URI_SCHEME = get_env('SQLALCHEMY_DATABASE_URI_SCHEME')
-
-        db_extras = f"?client_encoding={db_credentials['DB_CHARSET']}" if db_credentials['DB_CHARSET'] else ""
-
-        self.SQLALCHEMY_DATABASE_URI = f"{self.SQLALCHEMY_DATABASE_URI_SCHEME}://{db_credentials['DB_USERNAME']}:{db_credentials['DB_PASSWORD']}@{db_credentials['DB_HOST']}:{db_credentials['DB_PORT']}/{db_credentials['DB_DATABASE']}{db_extras}"
-        self.SQLALCHEMY_ENGINE_OPTIONS = {
-            'pool_size': int(get_env('SQLALCHEMY_POOL_SIZE')),
-            'max_overflow': int(get_env('SQLALCHEMY_MAX_OVERFLOW')),
-            'pool_recycle': int(get_env('SQLALCHEMY_POOL_RECYCLE')),
-            'pool_pre_ping': get_bool_env('SQLALCHEMY_POOL_PRE_PING'),
-            'connect_args': {'options': '-c timezone=UTC'},
-        }
-
-        self.SQLALCHEMY_ECHO = get_bool_env('SQLALCHEMY_ECHO')
-
-        # ------------------------
-        # Redis Configurations.
-        # ------------------------
-        self.REDIS_HOST = get_env('REDIS_HOST')
-        self.REDIS_PORT = get_env('REDIS_PORT')
-        self.REDIS_USERNAME = get_env('REDIS_USERNAME')
-        self.REDIS_PASSWORD = get_env('REDIS_PASSWORD')
-        self.REDIS_DB = get_env('REDIS_DB')
-        self.REDIS_USE_SSL = get_bool_env('REDIS_USE_SSL')
-
-        # ------------------------
-        # Celery worker Configurations.
-        # ------------------------
-        self.CELERY_BROKER_URL = get_env('CELERY_BROKER_URL')
-        self.CELERY_BACKEND = get_env('CELERY_BACKEND')
-        self.CELERY_RESULT_BACKEND = 'db+{}'.format(self.SQLALCHEMY_DATABASE_URI) \
-            if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL
-        self.BROKER_USE_SSL = self.CELERY_BROKER_URL.startswith('rediss://')
-
-        # ------------------------
-        # Code Execution Sandbox Configurations.
-        # ------------------------
-        self.CODE_EXECUTION_ENDPOINT = get_env('CODE_EXECUTION_ENDPOINT')
-        self.CODE_EXECUTION_API_KEY = get_env('CODE_EXECUTION_API_KEY')
-
-        # ------------------------
-        # File Storage Configurations.
-        # ------------------------
-        self.STORAGE_TYPE = get_env('STORAGE_TYPE')
-        self.STORAGE_LOCAL_PATH = get_env('STORAGE_LOCAL_PATH')
-
-        # S3 Storage settings
-        self.S3_USE_AWS_MANAGED_IAM = get_bool_env('S3_USE_AWS_MANAGED_IAM')
-        self.S3_ENDPOINT = get_env('S3_ENDPOINT')
-        self.S3_BUCKET_NAME = get_env('S3_BUCKET_NAME')
-        self.S3_ACCESS_KEY = get_env('S3_ACCESS_KEY')
-        self.S3_SECRET_KEY = get_env('S3_SECRET_KEY')
-        self.S3_REGION = get_env('S3_REGION')
-        self.S3_ADDRESS_STYLE = get_env('S3_ADDRESS_STYLE')
-
-        # Azure Blob Storage settings
-        self.AZURE_BLOB_ACCOUNT_NAME = get_env('AZURE_BLOB_ACCOUNT_NAME')
-        self.AZURE_BLOB_ACCOUNT_KEY = get_env('AZURE_BLOB_ACCOUNT_KEY')
-        self.AZURE_BLOB_CONTAINER_NAME = get_env('AZURE_BLOB_CONTAINER_NAME')
-        self.AZURE_BLOB_ACCOUNT_URL = get_env('AZURE_BLOB_ACCOUNT_URL')
-
-        # Aliyun Storage settings
-        self.ALIYUN_OSS_BUCKET_NAME = get_env('ALIYUN_OSS_BUCKET_NAME')
-        self.ALIYUN_OSS_ACCESS_KEY = get_env('ALIYUN_OSS_ACCESS_KEY')
-        self.ALIYUN_OSS_SECRET_KEY = get_env('ALIYUN_OSS_SECRET_KEY')
-        self.ALIYUN_OSS_ENDPOINT = get_env('ALIYUN_OSS_ENDPOINT')
-        self.ALIYUN_OSS_REGION = get_env('ALIYUN_OSS_REGION')
-        self.ALIYUN_OSS_AUTH_VERSION = get_env('ALIYUN_OSS_AUTH_VERSION')
-
-        # Google Cloud Storage settings
-        self.GOOGLE_STORAGE_BUCKET_NAME = get_env('GOOGLE_STORAGE_BUCKET_NAME')
-        self.GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64 = get_env('GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64')
-
-        # ------------------------
-        # Vector Store Configurations.
-        # Currently, only support: qdrant, milvus, zilliz, weaviate, relyt, pgvector
-        # ------------------------
-        self.VECTOR_STORE = get_env('VECTOR_STORE')
-        self.KEYWORD_STORE = get_env('KEYWORD_STORE')
-
-        # qdrant settings
-        self.QDRANT_URL = get_env('QDRANT_URL')
-        self.QDRANT_API_KEY = get_env('QDRANT_API_KEY')
-        self.QDRANT_CLIENT_TIMEOUT = get_env('QDRANT_CLIENT_TIMEOUT')
-        self.QDRANT_GRPC_ENABLED = get_env('QDRANT_GRPC_ENABLED')
-        self.QDRANT_GRPC_PORT = get_env('QDRANT_GRPC_PORT')
-
-        # milvus / zilliz setting
-        self.MILVUS_HOST = get_env('MILVUS_HOST')
-        self.MILVUS_PORT = get_env('MILVUS_PORT')
-        self.MILVUS_USER = get_env('MILVUS_USER')
-        self.MILVUS_PASSWORD = get_env('MILVUS_PASSWORD')
-        self.MILVUS_SECURE = get_env('MILVUS_SECURE')
-        self.MILVUS_DATABASE = get_env('MILVUS_DATABASE')
-
-        # weaviate settings
-        self.WEAVIATE_ENDPOINT = get_env('WEAVIATE_ENDPOINT')
-        self.WEAVIATE_API_KEY = get_env('WEAVIATE_API_KEY')
-        self.WEAVIATE_GRPC_ENABLED = get_bool_env('WEAVIATE_GRPC_ENABLED')
-        self.WEAVIATE_BATCH_SIZE = int(get_env('WEAVIATE_BATCH_SIZE'))
-
-        # relyt settings
-        self.RELYT_HOST = get_env('RELYT_HOST')
-        self.RELYT_PORT = get_env('RELYT_PORT')
-        self.RELYT_USER = get_env('RELYT_USER')
-        self.RELYT_PASSWORD = get_env('RELYT_PASSWORD')
-        self.RELYT_DATABASE = get_env('RELYT_DATABASE')
-
-
-        # tencent settings
-        self.TENCENT_VECTOR_DB_URL = get_env('TENCENT_VECTOR_DB_URL')
-        self.TENCENT_VECTOR_DB_API_KEY = get_env('TENCENT_VECTOR_DB_API_KEY')
-        self.TENCENT_VECTOR_DB_TIMEOUT = get_env('TENCENT_VECTOR_DB_TIMEOUT')
-        self.TENCENT_VECTOR_DB_USERNAME = get_env('TENCENT_VECTOR_DB_USERNAME')
-        self.TENCENT_VECTOR_DB_DATABASE = get_env('TENCENT_VECTOR_DB_DATABASE')
-        self.TENCENT_VECTOR_DB_SHARD = get_env('TENCENT_VECTOR_DB_SHARD')
-        self.TENCENT_VECTOR_DB_REPLICAS = get_env('TENCENT_VECTOR_DB_REPLICAS')
-
-        # pgvecto rs settings
-        self.PGVECTO_RS_HOST = get_env('PGVECTO_RS_HOST')
-        self.PGVECTO_RS_PORT = get_env('PGVECTO_RS_PORT')
-        self.PGVECTO_RS_USER = get_env('PGVECTO_RS_USER')
-        self.PGVECTO_RS_PASSWORD = get_env('PGVECTO_RS_PASSWORD')
-        self.PGVECTO_RS_DATABASE = get_env('PGVECTO_RS_DATABASE')
-
-        # pgvector settings
-        self.PGVECTOR_HOST = get_env('PGVECTOR_HOST')
-        self.PGVECTOR_PORT = get_env('PGVECTOR_PORT')
-        self.PGVECTOR_USER = get_env('PGVECTOR_USER')
-        self.PGVECTOR_PASSWORD = get_env('PGVECTOR_PASSWORD')
-        self.PGVECTOR_DATABASE = get_env('PGVECTOR_DATABASE')
-
-        # tidb-vector settings
-        self.TIDB_VECTOR_HOST = get_env('TIDB_VECTOR_HOST')
-        self.TIDB_VECTOR_PORT = get_env('TIDB_VECTOR_PORT')
-        self.TIDB_VECTOR_USER = get_env('TIDB_VECTOR_USER')
-        self.TIDB_VECTOR_PASSWORD = get_env('TIDB_VECTOR_PASSWORD')
-        self.TIDB_VECTOR_DATABASE = get_env('TIDB_VECTOR_DATABASE')
-
-        # chroma settings
-        self.CHROMA_HOST = get_env('CHROMA_HOST')
-        self.CHROMA_PORT = get_env('CHROMA_PORT')
-        self.CHROMA_TENANT = get_env('CHROMA_TENANT')
-        self.CHROMA_DATABASE = get_env('CHROMA_DATABASE')
-        self.CHROMA_AUTH_PROVIDER = get_env('CHROMA_AUTH_PROVIDER')
-        self.CHROMA_AUTH_CREDENTIALS = get_env('CHROMA_AUTH_CREDENTIALS')
-
-        # ------------------------
-        # Mail Configurations.
-        # ------------------------
-        self.MAIL_TYPE = get_env('MAIL_TYPE')
-        self.MAIL_DEFAULT_SEND_FROM = get_env('MAIL_DEFAULT_SEND_FROM')
-        self.RESEND_API_KEY = get_env('RESEND_API_KEY')
-        self.RESEND_API_URL = get_env('RESEND_API_URL')
-        # SMTP settings
-        self.SMTP_SERVER = get_env('SMTP_SERVER')
-        self.SMTP_PORT = get_env('SMTP_PORT')
-        self.SMTP_USERNAME = get_env('SMTP_USERNAME')
-        self.SMTP_PASSWORD = get_env('SMTP_PASSWORD')
-        self.SMTP_USE_TLS = get_bool_env('SMTP_USE_TLS')
-        self.SMTP_OPPORTUNISTIC_TLS = get_bool_env('SMTP_OPPORTUNISTIC_TLS')
-
-        # ------------------------
-        # Workspace Configurations.
-        # ------------------------
-        self.INVITE_EXPIRY_HOURS = int(get_env('INVITE_EXPIRY_HOURS'))
-
-        # ------------------------
-        # Sentry Configurations.
-        # ------------------------
-        self.SENTRY_DSN = get_env('SENTRY_DSN')
-        self.SENTRY_TRACES_SAMPLE_RATE = float(get_env('SENTRY_TRACES_SAMPLE_RATE'))
-        self.SENTRY_PROFILES_SAMPLE_RATE = float(get_env('SENTRY_PROFILES_SAMPLE_RATE'))
-
-        # ------------------------
-        # Business Configurations.
-        # ------------------------
-
-        # multi model send image format, support base64, url, default is base64
-        self.MULTIMODAL_SEND_IMAGE_FORMAT = get_env('MULTIMODAL_SEND_IMAGE_FORMAT')
-
-        # Dataset Configurations.
-        self.CLEAN_DAY_SETTING = get_env('CLEAN_DAY_SETTING')
-
-        # File upload Configurations.
-        self.UPLOAD_FILE_SIZE_LIMIT = int(get_env('UPLOAD_FILE_SIZE_LIMIT'))
-        self.UPLOAD_FILE_BATCH_LIMIT = int(get_env('UPLOAD_FILE_BATCH_LIMIT'))
-        self.UPLOAD_IMAGE_FILE_SIZE_LIMIT = int(get_env('UPLOAD_IMAGE_FILE_SIZE_LIMIT'))
-        self.BATCH_UPLOAD_LIMIT = get_env('BATCH_UPLOAD_LIMIT')
-
-        # RAG ETL Configurations.
-        self.ETL_TYPE = get_env('ETL_TYPE')
-        self.UNSTRUCTURED_API_URL = get_env('UNSTRUCTURED_API_URL')
-        self.UNSTRUCTURED_API_KEY = get_env('UNSTRUCTURED_API_KEY')
-        self.KEYWORD_DATA_SOURCE_TYPE = get_env('KEYWORD_DATA_SOURCE_TYPE')
-
-        # Indexing Configurations.
-        self.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH = get_env('INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH')
-
-        # Tool Configurations.
-        self.TOOL_ICON_CACHE_MAX_AGE = get_env('TOOL_ICON_CACHE_MAX_AGE')
-
-        self.WORKFLOW_MAX_EXECUTION_STEPS = int(get_env('WORKFLOW_MAX_EXECUTION_STEPS'))
-        self.WORKFLOW_MAX_EXECUTION_TIME = int(get_env('WORKFLOW_MAX_EXECUTION_TIME'))
-        self.WORKFLOW_CALL_MAX_DEPTH = int(get_env('WORKFLOW_CALL_MAX_DEPTH'))
-        self.APP_MAX_EXECUTION_TIME = int(get_env('APP_MAX_EXECUTION_TIME'))
-
-        # Moderation in app Configurations.
-        self.OUTPUT_MODERATION_BUFFER_SIZE = int(get_env('OUTPUT_MODERATION_BUFFER_SIZE'))
-
-        # Notion integration setting
-        self.NOTION_CLIENT_ID = get_env('NOTION_CLIENT_ID')
-        self.NOTION_CLIENT_SECRET = get_env('NOTION_CLIENT_SECRET')
-        self.NOTION_INTEGRATION_TYPE = get_env('NOTION_INTEGRATION_TYPE')
-        self.NOTION_INTERNAL_SECRET = get_env('NOTION_INTERNAL_SECRET')
-        self.NOTION_INTEGRATION_TOKEN = get_env('NOTION_INTEGRATION_TOKEN')
-
-        # ------------------------
-        # Platform Configurations.
-        # ------------------------
-        self.GITHUB_CLIENT_ID = get_env('GITHUB_CLIENT_ID')
-        self.GITHUB_CLIENT_SECRET = get_env('GITHUB_CLIENT_SECRET')
-        self.GOOGLE_CLIENT_ID = get_env('GOOGLE_CLIENT_ID')
-        self.GOOGLE_CLIENT_SECRET = get_env('GOOGLE_CLIENT_SECRET')
-        self.OAUTH_REDIRECT_PATH = get_env('OAUTH_REDIRECT_PATH')
-
-        self.HOSTED_OPENAI_API_KEY = get_env('HOSTED_OPENAI_API_KEY')
-        self.HOSTED_OPENAI_API_BASE = get_env('HOSTED_OPENAI_API_BASE')
-        self.HOSTED_OPENAI_API_ORGANIZATION = get_env('HOSTED_OPENAI_API_ORGANIZATION')
-        self.HOSTED_OPENAI_TRIAL_ENABLED = get_bool_env('HOSTED_OPENAI_TRIAL_ENABLED')
-        self.HOSTED_OPENAI_TRIAL_MODELS = get_env('HOSTED_OPENAI_TRIAL_MODELS')
-        self.HOSTED_OPENAI_QUOTA_LIMIT = int(get_env('HOSTED_OPENAI_QUOTA_LIMIT'))
-        self.HOSTED_OPENAI_PAID_ENABLED = get_bool_env('HOSTED_OPENAI_PAID_ENABLED')
-        self.HOSTED_OPENAI_PAID_MODELS = get_env('HOSTED_OPENAI_PAID_MODELS')
-
-        self.HOSTED_AZURE_OPENAI_ENABLED = get_bool_env('HOSTED_AZURE_OPENAI_ENABLED')
-        self.HOSTED_AZURE_OPENAI_API_KEY = get_env('HOSTED_AZURE_OPENAI_API_KEY')
-        self.HOSTED_AZURE_OPENAI_API_BASE = get_env('HOSTED_AZURE_OPENAI_API_BASE')
-        self.HOSTED_AZURE_OPENAI_QUOTA_LIMIT = int(get_env('HOSTED_AZURE_OPENAI_QUOTA_LIMIT'))
-
-        self.HOSTED_ANTHROPIC_API_BASE = get_env('HOSTED_ANTHROPIC_API_BASE')
-        self.HOSTED_ANTHROPIC_API_KEY = get_env('HOSTED_ANTHROPIC_API_KEY')
-        self.HOSTED_ANTHROPIC_TRIAL_ENABLED = get_bool_env('HOSTED_ANTHROPIC_TRIAL_ENABLED')
-        self.HOSTED_ANTHROPIC_QUOTA_LIMIT = int(get_env('HOSTED_ANTHROPIC_QUOTA_LIMIT'))
-        self.HOSTED_ANTHROPIC_PAID_ENABLED = get_bool_env('HOSTED_ANTHROPIC_PAID_ENABLED')
-
-        self.HOSTED_MINIMAX_ENABLED = get_bool_env('HOSTED_MINIMAX_ENABLED')
-        self.HOSTED_SPARK_ENABLED = get_bool_env('HOSTED_SPARK_ENABLED')
-        self.HOSTED_ZHIPUAI_ENABLED = get_bool_env('HOSTED_ZHIPUAI_ENABLED')
-
-        self.HOSTED_MODERATION_ENABLED = get_bool_env('HOSTED_MODERATION_ENABLED')
-        self.HOSTED_MODERATION_PROVIDERS = get_env('HOSTED_MODERATION_PROVIDERS')
-
-        # fetch app templates mode, remote, builtin, db(only for dify SaaS), default: remote
-        self.HOSTED_FETCH_APP_TEMPLATES_MODE = get_env('HOSTED_FETCH_APP_TEMPLATES_MODE')
-        self.HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN = get_env('HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN')
-
-        # Model Load Balancing Configurations.
-        self.MODEL_LB_ENABLED = get_bool_env('MODEL_LB_ENABLED')
-
-        # Platform Billing Configurations.
-        self.BILLING_ENABLED = get_bool_env('BILLING_ENABLED')
-
-        # ------------------------
-        # Enterprise feature Configurations.
-        # **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
-        # ------------------------
-        self.ENTERPRISE_ENABLED = get_bool_env('ENTERPRISE_ENABLED')
-        self.CAN_REPLACE_LOGO = get_bool_env('CAN_REPLACE_LOGO')

api/configs/app_config.py

@@ -0,0 +1,43 @@
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from configs.deploy import DeploymentConfig
+from configs.enterprise import EnterpriseFeatureConfig
+from configs.extra import ExtraServiceConfig
+from configs.feature import FeatureConfig
+from configs.middleware import MiddlewareConfig
+from configs.packaging import PackagingInfo
+
+
+class DifyConfig(
+    # based on pydantic-settings
+    BaseSettings,
+
+    # Packaging info
+    PackagingInfo,
+
+    # Deployment configs
+    DeploymentConfig,
+
+    # Feature configs
+    FeatureConfig,
+
+    # Middleware configs
+    MiddlewareConfig,
+
+    # Extra service configs
+    ExtraServiceConfig,
+
+    # Enterprise feature configs
+    # **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
+    EnterpriseFeatureConfig,
+):
+
+    model_config = SettingsConfigDict(
+        # read from dotenv format config file
+        env_file='.env',
+        env_file_encoding='utf-8',
+        env_ignore_empty=True,
+
+        # ignore extra attributes
+        extra='ignore',
+    )

api/configs/deploy/__init__.py

@@ -0,0 +1,26 @@
+from pydantic import BaseModel, Field
+
+
+class DeploymentConfig(BaseModel):
+    """
+    Deployment configs
+    """
+    APPLICATION_NAME: str = Field(
+        description='application name',
+        default='langgenius/dify',
+    )
+
+    TESTING: bool = Field(
+        description='',
+        default=False,
+    )
+
+    EDITION: str = Field(
+        description='deployment edition',
+        default='SELF_HOSTED',
+    )
+
+    DEPLOY_ENV: str = Field(
+        description='deployment environment, default to PRODUCTION.',
+        default='PRODUCTION',
+    )

api/configs/enterprise/__init__.py

@@ -0,0 +1,18 @@
+from pydantic import BaseModel, Field
+
+
+class EnterpriseFeatureConfig(BaseModel):
+    """
+    Enterprise feature configs.
+    **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
+    """
+    ENTERPRISE_ENABLED: bool = Field(
+        description='whether to enable enterprise features.'
+                    'Before using, please contact business@dify.ai by email to inquire about licensing matters.',
+        default=False,
+    )
+
+    CAN_REPLACE_LOGO: bool = Field(
+        description='whether to allow replacing enterprise logo.',
+        default=False,
+    )

api/configs/extra/__init__.py

@@ -0,0 +1,12 @@
+from pydantic import BaseModel
+
+from configs.extra.notion_config import NotionConfig
+from configs.extra.sentry_config import SentryConfig
+
+
+class ExtraServiceConfig(
+    # place the configs in alphabet order
+    NotionConfig,
+    SentryConfig,
+):
+    pass

api/configs/extra/notion_config.py

@@ -0,0 +1,33 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class NotionConfig(BaseModel):
+    """
+    Notion integration configs
+    """
+    NOTION_CLIENT_ID: Optional[str] = Field(
+        description='Notion client ID',
+        default=None,
+    )
+
+    NOTION_CLIENT_SECRET: Optional[str] = Field(
+        description='Notion client secret key',
+        default=None,
+    )
+
+    NOTION_INTEGRATION_TYPE: Optional[str] = Field(
+        description='Notion integration type, default to None, available values: internal.',
+        default=None,
+    )
+
+    NOTION_INTERNAL_SECRET: Optional[str] = Field(
+        description='Notion internal secret key',
+        default=None,
+    )
+
+    NOTION_INTEGRATION_TOKEN: Optional[str] = Field(
+        description='Notion integration token',
+        default=None,
+    )

api/configs/extra/sentry_config.py

@@ -0,0 +1,23 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, NonNegativeFloat
+
+
+class SentryConfig(BaseModel):
+    """
+    Sentry configs
+    """
+    SENTRY_DSN: Optional[str] = Field(
+        description='Sentry DSN',
+        default=None,
+    )
+
+    SENTRY_TRACES_SAMPLE_RATE: NonNegativeFloat = Field(
+        description='Sentry trace sample rate',
+        default=1.0,
+    )
+
+    SENTRY_PROFILES_SAMPLE_RATE: NonNegativeFloat = Field(
+        description='Sentry profiles sample rate',
+        default=1.0,
+    )

api/configs/feature/__init__.py

@@ -0,0 +1,452 @@
+from typing import Optional
+
+from pydantic import AliasChoices, BaseModel, Field, NonNegativeInt, PositiveInt, computed_field
+
+from configs.feature.hosted_service import HostedServiceConfig
+
+
+class SecurityConfig(BaseModel):
+    """
+    Secret Key configs
+    """
+    SECRET_KEY: Optional[str] = Field(
+        description='Your App secret key will be used for securely signing the session cookie'
+                    'Make sure you are changing this key for your deployment with a strong key.'
+                    'You can generate a strong key using `openssl rand -base64 42`.'
+                    'Alternatively you can set it with `SECRET_KEY` environment variable.',
+        default=None,
+    )
+
+
+class AppExecutionConfig(BaseModel):
+    """
+    App Execution configs
+    """
+    APP_MAX_EXECUTION_TIME: PositiveInt = Field(
+        description='execution timeout in seconds for app execution',
+        default=1200,
+    )
+
+
+class CodeExecutionSandboxConfig(BaseModel):
+    """
+    Code Execution Sandbox configs
+    """
+    CODE_EXECUTION_ENDPOINT: str = Field(
+        description='endpoint URL of code execution servcie',
+        default='http://sandbox:8194',
+    )
+
+    CODE_EXECUTION_API_KEY: str = Field(
+        description='API key for code execution service',
+        default='dify-sandbox',
+    )
+
+
+class EndpointConfig(BaseModel):
+    """
+    Module URL configs
+    """
+    CONSOLE_API_URL: str = Field(
+        description='The backend URL prefix of the console API.'
+                    'used to concatenate the login authorization callback or notion integration callback.',
+        default='https://cloud.dify.ai',
+    )
+
+    CONSOLE_WEB_URL: str = Field(
+        description='The front-end URL prefix of the console web.'
+                    'used to concatenate some front-end addresses and for CORS configuration use.',
+        default='https://cloud.dify.ai',
+    )
+
+    SERVICE_API_URL: str = Field(
+        description='Service API Url prefix.'
+                    'used to display Service API Base Url to the front-end.',
+        default='https://api.dify.ai',
+    )
+
+    APP_WEB_URL: str = Field(
+        description='WebApp Url prefix.'
+                    'used to display WebAPP API Base Url to the front-end.',
+        default='https://udify.app',
+    )
+
+
+class FileAccessConfig(BaseModel):
+    """
+    File Access configs
+    """
+    FILES_URL: str = Field(
+        description='File preview or download Url prefix.'
+                    ' used to display File preview or download Url to the front-end or as Multi-model inputs;'
+                    'Url is signed and has expiration time.',
+        validation_alias=AliasChoices('FILES_URL', 'CONSOLE_API_URL'),
+        alias_priority=1,
+        default='https://cloud.dify.ai',
+    )
+
+    FILES_ACCESS_TIMEOUT: int = Field(
+        description='timeout in seconds for file accessing',
+        default=300,
+    )
+
+
+class FileUploadConfig(BaseModel):
+    """
+    File Uploading configs
+    """
+    UPLOAD_FILE_SIZE_LIMIT: NonNegativeInt = Field(
+        description='size limit in Megabytes for uploading files',
+        default=15,
+    )
+
+    UPLOAD_FILE_BATCH_LIMIT: NonNegativeInt = Field(
+        description='batch size limit for uploading files',
+        default=5,
+    )
+
+    UPLOAD_IMAGE_FILE_SIZE_LIMIT: NonNegativeInt = Field(
+        description='image file size limit in Megabytes for uploading files',
+        default=10,
+    )
+
+    BATCH_UPLOAD_LIMIT: NonNegativeInt = Field(
+        description='',  # todo: to be clarified
+        default=20,
+    )
+
+
+class HttpConfig(BaseModel):
+    """
+    HTTP configs
+    """
+    API_COMPRESSION_ENABLED: bool = Field(
+        description='whether to enable HTTP response compression of gzip',
+        default=False,
+    )
+
+    inner_CONSOLE_CORS_ALLOW_ORIGINS: str = Field(
+        description='',
+        validation_alias=AliasChoices('CONSOLE_CORS_ALLOW_ORIGINS', 'CONSOLE_WEB_URL'),
+        default='',
+    )
+
+    @computed_field
+    @property
+    def CONSOLE_CORS_ALLOW_ORIGINS(self) -> list[str]:
+        return self.inner_CONSOLE_CORS_ALLOW_ORIGINS.split(',')
+
+    inner_WEB_API_CORS_ALLOW_ORIGINS: Optional[str] = Field(
+        description='',
+        validation_alias=AliasChoices('WEB_API_CORS_ALLOW_ORIGINS'),
+        default='*',
+    )
+
+    @computed_field
+    @property
+    def WEB_API_CORS_ALLOW_ORIGINS(self) -> list[str]:
+        return self.inner_WEB_API_CORS_ALLOW_ORIGINS.split(',')
+
+
+class InnerAPIConfig(BaseModel):
+    """
+    Inner API configs
+    """
+    INNER_API: bool = Field(
+        description='whether to enable the inner API',
+        default=False,
+    )
+
+    INNER_API_KEY: Optional[str] = Field(
+        description='The inner API key is used to authenticate the inner API',
+        default=None,
+    )
+
+
+class LoggingConfig(BaseModel):
+    """
+    Logging configs
+    """
+
+    LOG_LEVEL: str = Field(
+        description='Log output level, default to INFO.'
+                    'It is recommended to set it to ERROR for production.',
+        default='INFO',
+    )
+
+    LOG_FILE: Optional[str] = Field(
+        description='logging output file path',
+        default=None,
+    )
+
+    LOG_FORMAT: str = Field(
+        description='log format',
+        default='%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s',
+    )
+
+    LOG_DATEFORMAT: Optional[str] = Field(
+        description='log date format',
+        default=None,
+    )
+
+    LOG_TZ: Optional[str] = Field(
+        description='specify log timezone, eg: America/New_York',
+        default=None,
+    )
+
+
+class ModelLoadBalanceConfig(BaseModel):
+    """
+    Model load balance configs
+    """
+    MODEL_LB_ENABLED: bool = Field(
+        description='whether to enable model load balancing',
+        default=False,
+    )
+
+
+class BillingConfig(BaseModel):
+    """
+    Platform Billing Configurations
+    """
+    BILLING_ENABLED: bool = Field(
+        description='whether to enable billing',
+        default=False,
+    )
+
+
+class UpdateConfig(BaseModel):
+    """
+    Update configs
+    """
+    CHECK_UPDATE_URL: str = Field(
+        description='url for checking updates',
+        default='https://updates.dify.ai',
+    )
+
+
+class WorkflowConfig(BaseModel):
+    """
+    Workflow feature configs
+    """
+
+    WORKFLOW_MAX_EXECUTION_STEPS: PositiveInt = Field(
+        description='max execution steps in single workflow execution',
+        default=500,
+    )
+
+    WORKFLOW_MAX_EXECUTION_TIME: PositiveInt = Field(
+        description='max execution time in seconds in single workflow execution',
+        default=1200,
+    )
+
+    WORKFLOW_CALL_MAX_DEPTH: PositiveInt = Field(
+        description='max depth of calling in single workflow execution',
+        default=5,
+    )
+
+
+class OAuthConfig(BaseModel):
+    """
+    oauth configs
+    """
+    OAUTH_REDIRECT_PATH: str = Field(
+        description='redirect path for OAuth',
+        default='/console/api/oauth/authorize',
+    )
+
+    GITHUB_CLIENT_ID: Optional[str] = Field(
+        description='GitHub client id for OAuth',
+        default=None,
+    )
+
+    GITHUB_CLIENT_SECRET: Optional[str] = Field(
+        description='GitHub client secret key for OAuth',
+        default=None,
+    )
+
+    GOOGLE_CLIENT_ID: Optional[str] = Field(
+        description='Google client id for OAuth',
+        default=None,
+    )
+
+    GOOGLE_CLIENT_SECRET: Optional[str] = Field(
+        description='Google client secret key for OAuth',
+        default=None,
+    )
+
+
+class ModerationConfig(BaseModel):
+    """
+    Moderation in app configs.
+    """
+
+    # todo: to be clarified in usage and unit
+    OUTPUT_MODERATION_BUFFER_SIZE: PositiveInt = Field(
+        description='buffer size for moderation',
+        default=300,
+    )
+
+
+class ToolConfig(BaseModel):
+    """
+    Tool configs
+    """
+
+    TOOL_ICON_CACHE_MAX_AGE: PositiveInt = Field(
+        description='max age in seconds for tool icon caching',
+        default=3600,
+    )
+
+
+class MailConfig(BaseModel):
+    """
+    Mail Configurations
+    """
+
+    MAIL_TYPE: Optional[str] = Field(
+        description='Mail provider type name, default to None, availabile values are `smtp` and `resend`.',
+        default=None,
+    )
+
+    MAIL_DEFAULT_SEND_FROM: Optional[str] = Field(
+        description='default email address for sending from ',
+        default=None,
+    )
+
+    RESEND_API_KEY: Optional[str] = Field(
+        description='API key for Resend',
+        default=None,
+    )
+
+    RESEND_API_URL: Optional[str] = Field(
+        description='API URL for Resend',
+        default=None,
+    )
+
+    SMTP_SERVER: Optional[str] = Field(
+        description='smtp server host',
+        default=None,
+    )
+
+    SMTP_PORT: Optional[int] = Field(
+        description='smtp server port',
+        default=465,
+    )
+
+    SMTP_USERNAME: Optional[str] = Field(
+        description='smtp server username',
+        default=None,
+    )
+
+    SMTP_PASSWORD: Optional[str] = Field(
+        description='smtp server password',
+        default=None,
+    )
+
+    SMTP_USE_TLS: bool = Field(
+        description='whether to use TLS connection to smtp server',
+        default=False,
+    )
+
+    SMTP_OPPORTUNISTIC_TLS: bool = Field(
+        description='whether to use opportunistic TLS connection to smtp server',
+        default=False,
+    )
+
+
+class RagEtlConfig(BaseModel):
+    """
+    RAG ETL Configurations.
+    """
+
+    ETL_TYPE: str = Field(
+        description='RAG ETL type name, default to `dify`, available values are `dify` and `Unstructured`. ',
+        default='dify',
+    )
+
+    KEYWORD_DATA_SOURCE_TYPE: str = Field(
+        description='source type for keyword data, default to `database`, available values are `database` .',
+        default='database',
+    )
+
+    UNSTRUCTURED_API_URL: Optional[str] = Field(
+        description='API URL for Unstructured',
+        default=None,
+    )
+
+    UNSTRUCTURED_API_KEY: Optional[str] = Field(
+        description='API key for Unstructured',
+        default=None,
+    )
+
+
+class DataSetConfig(BaseModel):
+    """
+    Dataset configs
+    """
+
+    CLEAN_DAY_SETTING: PositiveInt = Field(
+        description='interval in days for cleaning up dataset',
+        default=30,
+    )
+
+
+class WorkspaceConfig(BaseModel):
+    """
+    Workspace configs
+    """
+
+    INVITE_EXPIRY_HOURS: PositiveInt = Field(
+        description='workspaces invitation expiration in hours',
+        default=72,
+    )
+
+
+class IndexingConfig(BaseModel):
+    """
+    Indexing configs.
+    """
+
+    INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: PositiveInt = Field(
+        description='max segmentation token length for indexing',
+        default=1000,
+    )
+
+
+class ImageFormatConfig(BaseModel):
+    MULTIMODAL_SEND_IMAGE_FORMAT: str = Field(
+        description='multi model send image format, support base64, url, default is base64',
+        default='base64',
+    )
+
+
+class FeatureConfig(
+    # place the configs in alphabet order
+    AppExecutionConfig,
+    BillingConfig,
+    CodeExecutionSandboxConfig,
+    DataSetConfig,
+    EndpointConfig,
+    FileAccessConfig,
+    FileUploadConfig,
+    HttpConfig,
+    ImageFormatConfig,
+    InnerAPIConfig,
+    IndexingConfig,
+    LoggingConfig,
+    MailConfig,
+    ModelLoadBalanceConfig,
+    ModerationConfig,
+    OAuthConfig,
+    RagEtlConfig,
+    SecurityConfig,
+    ToolConfig,
+    UpdateConfig,
+    WorkflowConfig,
+    WorkspaceConfig,
+
+    # hosted services config
+    HostedServiceConfig,
+):
+    pass

api/configs/feature/hosted_service/__init__.py

@@ -0,0 +1,208 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, NonNegativeInt
+
+
+class HostedOpenAiConfig(BaseModel):
+    """
+    Hosted OpenAI service config
+    """
+
+    HOSTED_OPENAI_API_KEY: Optional[str] = Field(
+        description='',
+        default=None,
+    )
+
+    HOSTED_OPENAI_API_BASE: Optional[str] = Field(
+        description='',
+        default=None,
+    )
+
+    HOSTED_OPENAI_API_ORGANIZATION: Optional[str] = Field(
+        description='',
+        default=None,
+    )
+
+    HOSTED_OPENAI_TRIAL_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+    HOSTED_OPENAI_TRIAL_MODELS: str = Field(
+        description='',
+        default='gpt-3.5-turbo,'
+                'gpt-3.5-turbo-1106,'
+                'gpt-3.5-turbo-instruct,'
+                'gpt-3.5-turbo-16k,'
+                'gpt-3.5-turbo-16k-0613,'
+                'gpt-3.5-turbo-0613,'
+                'gpt-3.5-turbo-0125,'
+                'text-davinci-003',
+    )
+
+    HOSTED_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field(
+        description='',
+        default=200,
+    )
+
+    HOSTED_OPENAI_PAID_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+    HOSTED_OPENAI_PAID_MODELS: str = Field(
+        description='',
+        default='gpt-4,'
+                'gpt-4-turbo-preview,'
+                'gpt-4-turbo-2024-04-09,'
+                'gpt-4-1106-preview,'
+                'gpt-4-0125-preview,'
+                'gpt-3.5-turbo,'
+                'gpt-3.5-turbo-16k,'
+                'gpt-3.5-turbo-16k-0613,'
+                'gpt-3.5-turbo-1106,'
+                'gpt-3.5-turbo-0613,'
+                'gpt-3.5-turbo-0125,'
+                'gpt-3.5-turbo-instruct,'
+                'text-davinci-003',
+    )
+
+
+class HostedAzureOpenAiConfig(BaseModel):
+    """
+    Hosted OpenAI service config
+    """
+
+    HOSTED_AZURE_OPENAI_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+    HOSTED_OPENAI_API_KEY: Optional[str] = Field(
+        description='',
+        default=None,
+    )
+
+    HOSTED_AZURE_OPENAI_API_BASE: Optional[str] = Field(
+        description='',
+        default=None,
+    )
+
+    HOSTED_AZURE_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field(
+        description='',
+        default=200,
+    )
+
+
+class HostedAnthropicConfig(BaseModel):
+    """
+    Hosted Azure OpenAI service config
+    """
+
+    HOSTED_ANTHROPIC_API_BASE: Optional[str] = Field(
+        description='',
+        default=None,
+    )
+
+    HOSTED_ANTHROPIC_API_KEY: Optional[str] = Field(
+        description='',
+        default=None,
+    )
+
+    HOSTED_ANTHROPIC_TRIAL_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+    HOSTED_ANTHROPIC_QUOTA_LIMIT: NonNegativeInt = Field(
+        description='',
+        default=600000,
+    )
+
+    HOSTED_ANTHROPIC_PAID_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+
+class HostedMinmaxConfig(BaseModel):
+    """
+    Hosted Minmax service config
+    """
+
+    HOSTED_MINIMAX_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+
+class HostedSparkConfig(BaseModel):
+    """
+    Hosted Spark service config
+    """
+
+    HOSTED_SPARK_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+
+class HostedZhipuAIConfig(BaseModel):
+    """
+    Hosted Minmax service config
+    """
+
+    HOSTED_ZHIPUAI_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+
+class HostedModerationConfig(BaseModel):
+    """
+    Hosted Moderation service config
+    """
+
+    HOSTED_MODERATION_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+    HOSTED_MODERATION_PROVIDERS: str = Field(
+        description='',
+        default='',
+    )
+
+
+class HostedFetchAppTemplateConfig(BaseModel):
+    """
+    Hosted Moderation service config
+    """
+
+    HOSTED_FETCH_APP_TEMPLATES_MODE: str = Field(
+        description='the mode for fetching app templates,'
+                    ' default to remote,'
+                    ' available values: remote, db, builtin',
+        default='remote',
+    )
+
+    HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN: str = Field(
+        description='the domain for fetching remote app templates',
+        default='https://tmpl.dify.ai',
+    )
+
+
+class HostedServiceConfig(
+    # place the configs in alphabet order
+    HostedAnthropicConfig,
+    HostedAzureOpenAiConfig,
+    HostedFetchAppTemplateConfig,
+    HostedMinmaxConfig,
+    HostedOpenAiConfig,
+    HostedSparkConfig,
+    HostedZhipuAIConfig,
+
+    # moderation
+    HostedModerationConfig,
+):
+    pass

api/configs/middleware/__init__.py

@@ -0,0 +1,185 @@
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field, NonNegativeInt, PositiveInt, computed_field
+
+from configs.middleware.cache.redis_config import RedisConfig
+from configs.middleware.storage.aliyun_oss_storage_config import AliyunOSSStorageConfig
+from configs.middleware.storage.amazon_s3_storage_config import S3StorageConfig
+from configs.middleware.storage.azure_blob_storage_config import AzureBlobStorageConfig
+from configs.middleware.storage.google_cloud_storage_config import GoogleCloudStorageConfig
+from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
+from configs.middleware.vdb.chroma_config import ChromaConfig
+from configs.middleware.vdb.milvus_config import MilvusConfig
+from configs.middleware.vdb.opensearch_config import OpenSearchConfig
+from configs.middleware.vdb.oracle_config import OracleConfig
+from configs.middleware.vdb.pgvector_config import PGVectorConfig
+from configs.middleware.vdb.pgvectors_config import PGVectoRSConfig
+from configs.middleware.vdb.qdrant_config import QdrantConfig
+from configs.middleware.vdb.relyt_config import RelytConfig
+from configs.middleware.vdb.tencent_vector_config import TencentVectorDBConfig
+from configs.middleware.vdb.tidb_vector_config import TiDBVectorConfig
+from configs.middleware.vdb.weaviate_config import WeaviateConfig
+
+
+class StorageConfig(BaseModel):
+    STORAGE_TYPE: str = Field(
+        description='storage type,'
+                    ' default to `local`,'
+                    ' available values are `local`, `s3`, `azure-blob`, `aliyun-oss`, `google-storage`.',
+        default='local',
+    )
+
+    STORAGE_LOCAL_PATH: str = Field(
+        description='local storage path',
+        default='storage',
+    )
+
+
+class VectorStoreConfig(BaseModel):
+    VECTOR_STORE: Optional[str] = Field(
+        description='vector store type',
+        default=None,
+    )
+
+
+class KeywordStoreConfig(BaseModel):
+    KEYWORD_STORE: str = Field(
+        description='keyword store type',
+        default='jieba',
+    )
+
+
+class DatabaseConfig:
+    DB_HOST: str = Field(
+        description='db host',
+        default='localhost',
+    )
+
+    DB_PORT: PositiveInt = Field(
+        description='db port',
+        default=5432,
+    )
+
+    DB_USERNAME: str = Field(
+        description='db username',
+        default='postgres',
+    )
+
+    DB_PASSWORD: str = Field(
+        description='db password',
+        default='',
+    )
+
+    DB_DATABASE: str = Field(
+        description='db database',
+        default='dify',
+    )
+
+    DB_CHARSET: str = Field(
+        description='db charset',
+        default='',
+    )
+
+    SQLALCHEMY_DATABASE_URI_SCHEME: str = Field(
+        description='db uri scheme',
+        default='postgresql',
+    )
+
+    @computed_field
+    @property
+    def SQLALCHEMY_DATABASE_URI(self) -> str:
+        db_extras = f"?client_encoding={self.DB_CHARSET}" if self.DB_CHARSET else ""
+        return (f"{self.SQLALCHEMY_DATABASE_URI_SCHEME}://"
+                f"{self.DB_USERNAME}:{self.DB_PASSWORD}@{self.DB_HOST}:{self.DB_PORT}/{self.DB_DATABASE}"
+                f"{db_extras}")
+
+    SQLALCHEMY_POOL_SIZE: NonNegativeInt = Field(
+        description='pool size of SqlAlchemy',
+        default=30,
+    )
+
+    SQLALCHEMY_MAX_OVERFLOW: NonNegativeInt = Field(
+        description='max overflows for SqlAlchemy',
+        default=10,
+    )
+
+    SQLALCHEMY_POOL_RECYCLE: NonNegativeInt = Field(
+        description='SqlAlchemy pool recycle',
+        default=3600,
+    )
+
+    SQLALCHEMY_POOL_PRE_PING: bool = Field(
+        description='whether to enable pool pre-ping in SqlAlchemy',
+        default=False,
+    )
+
+    SQLALCHEMY_ECHO: bool = Field(
+        description='whether to enable SqlAlchemy echo',
+        default=False,
+    )
+
+    @computed_field
+    @property
+    def SQLALCHEMY_ENGINE_OPTIONS(self) -> dict[str, Any]:
+        return {
+            'pool_size': self.SQLALCHEMY_POOL_SIZE,
+            'max_overflow': self.SQLALCHEMY_MAX_OVERFLOW,
+            'pool_recycle': self.SQLALCHEMY_POOL_RECYCLE,
+            'pool_pre_ping': self.SQLALCHEMY_POOL_PRE_PING,
+            'connect_args': {'options': '-c timezone=UTC'},
+        }
+
+
+class CeleryConfig(DatabaseConfig):
+    CELERY_BACKEND: str = Field(
+        description='Celery backend, available values are `database`, `redis`',
+        default='database',
+    )
+
+    CELERY_BROKER_URL: Optional[str] = Field(
+        description='CELERY_BROKER_URL',
+        default=None,
+    )
+
+    @computed_field
+    @property
+    def CELERY_RESULT_BACKEND(self) -> str:
+        return 'db+{}'.format(self.SQLALCHEMY_DATABASE_URI) \
+            if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL
+
+    @computed_field
+    @property
+    def BROKER_USE_SSL(self) -> bool:
+        return self.CELERY_BROKER_URL.startswith('rediss://') if self.CELERY_BROKER_URL else False
+
+
+class MiddlewareConfig(
+    # place the configs in alphabet order
+    CeleryConfig,
+    DatabaseConfig,
+    KeywordStoreConfig,
+    RedisConfig,
+
+    # configs of storage and storage providers
+    StorageConfig,
+    AliyunOSSStorageConfig,
+    AzureBlobStorageConfig,
+    GoogleCloudStorageConfig,
+    TencentCloudCOSStorageConfig,
+    S3StorageConfig,
+
+    # configs of vdb and vdb providers
+    VectorStoreConfig,
+    ChromaConfig,
+    MilvusConfig,
+    OpenSearchConfig,
+    OracleConfig,
+    PGVectorConfig,
+    PGVectoRSConfig,
+    QdrantConfig,
+    RelytConfig,
+    TencentVectorDBConfig,
+    TiDBVectorConfig,
+    WeaviateConfig,
+):
+    pass

api/configs/middleware/cache/redis_config.py

@@ -0,0 +1,38 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, NonNegativeInt, PositiveInt
+
+
+class RedisConfig(BaseModel):
+    """
+    Redis configs
+    """
+    REDIS_HOST: str = Field(
+        description='Redis host',
+        default='localhost',
+    )
+
+    REDIS_PORT: PositiveInt = Field(
+        description='Redis port',
+        default=6379,
+    )
+
+    REDIS_USERNAME: Optional[str] = Field(
+        description='Redis username',
+        default=None,
+    )
+
+    REDIS_PASSWORD: Optional[str] = Field(
+        description='Redis password',
+        default=None,
+    )
+
+    REDIS_DB: NonNegativeInt = Field(
+        description='Redis database id, default to 0',
+        default=0,
+    )
+
+    REDIS_USE_SSL: bool = Field(
+        description='whether to use SSL for Redis connection',
+        default=False,
+    )

api/configs/middleware/storage/aliyun_oss_storage_config.py

@@ -0,0 +1,39 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class AliyunOSSStorageConfig(BaseModel):
+    """
+    Aliyun storage configs
+    """
+
+    ALIYUN_OSS_BUCKET_NAME: Optional[str] = Field(
+        description='Aliyun OSS bucket name',
+        default=None,
+    )
+
+    ALIYUN_OSS_ACCESS_KEY: Optional[str] = Field(
+        description='Aliyun OSS access key',
+        default=None,
+    )
+
+    ALIYUN_OSS_SECRET_KEY: Optional[str] = Field(
+        description='Aliyun OSS secret key',
+        default=None,
+    )
+
+    ALIYUN_OSS_ENDPOINT: Optional[str] = Field(
+        description='Aliyun OSS endpoint URL',
+        default=None,
+    )
+
+    ALIYUN_OSS_REGION: Optional[str] = Field(
+        description='Aliyun OSS region',
+        default=None,
+    )
+
+    ALIYUN_OSS_AUTH_VERSION: Optional[str] = Field(
+        description='Aliyun OSS authentication version',
+        default=None,
+    )

api/configs/middleware/storage/amazon_s3_storage_config.py

@@ -0,0 +1,44 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class S3StorageConfig(BaseModel):
+    """
+    S3 storage configs
+    """
+
+    S3_ENDPOINT: Optional[str] = Field(
+        description='S3 storage endpoint',
+        default=None,
+    )
+
+    S3_REGION: Optional[str] = Field(
+        description='S3 storage region',
+        default=None,
+    )
+
+    S3_BUCKET_NAME: Optional[str] = Field(
+        description='S3 storage bucket name',
+        default=None,
+    )
+
+    S3_ACCESS_KEY: Optional[str] = Field(
+        description='S3 storage access key',
+        default=None,
+    )
+
+    S3_SECRET_KEY: Optional[str] = Field(
+        description='S3 storage secret key',
+        default=None,
+    )
+
+    S3_ADDRESS_STYLE: str = Field(
+        description='S3 storage address style',
+        default='auto',
+    )
+
+    S3_USE_AWS_MANAGED_IAM: bool = Field(
+        description='whether to use aws managed IAM for S3',
+        default=False,
+    )

api/configs/middleware/storage/azure_blob_storage_config.py

@@ -0,0 +1,29 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class AzureBlobStorageConfig(BaseModel):
+    """
+    Azure Blob storage configs
+    """
+
+    AZURE_BLOB_ACCOUNT_NAME: Optional[str] = Field(
+        description='Azure Blob account name',
+        default=None,
+    )
+
+    AZURE_BLOB_ACCOUNT_KEY: Optional[str] = Field(
+        description='Azure Blob account key',
+        default=None,
+    )
+
+    AZURE_BLOB_CONTAINER_NAME: Optional[str] = Field(
+        description='Azure Blob container name',
+        default=None,
+    )
+
+    AZURE_BLOB_ACCOUNT_URL: Optional[str] = Field(
+        description='Azure Blob account URL',
+        default=None,
+    )

api/configs/middleware/storage/google_cloud_storage_config.py

@@ -0,0 +1,19 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class GoogleCloudStorageConfig(BaseModel):
+    """
+    Google Cloud storage configs
+    """
+
+    GOOGLE_STORAGE_BUCKET_NAME: Optional[str] = Field(
+        description='Google Cloud storage bucket name',
+        default=None,
+    )
+
+    GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: Optional[str] = Field(
+        description='Google Cloud storage service account json base64',
+        default=None,
+    )

api/configs/middleware/storage/tencent_cos_storage_config.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class TencentCloudCOSStorageConfig(BaseModel):
+    """
+    Tencent Cloud COS storage configs
+    """
+
+    TENCENT_COS_BUCKET_NAME: Optional[str] = Field(
+        description='Tencent Cloud COS bucket name',
+        default=None,
+    )
+
+    TENCENT_COS_REGION: Optional[str] = Field(
+        description='Tencent Cloud COS region',
+        default=None,
+    )
+
+    TENCENT_COS_SECRET_ID: Optional[str] = Field(
+        description='Tencent Cloud COS secret id',
+        default=None,
+    )
+
+    TENCENT_COS_SECRET_KEY: Optional[str] = Field(
+        description='Tencent Cloud COS secret key',
+        default=None,
+    )
+
+    TENCENT_COS_SCHEME: Optional[str] = Field(
+        description='Tencent Cloud COS scheme',
+        default=None,
+    )

api/configs/middleware/vdb/chroma_config.py

@@ -0,0 +1,39 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class ChromaConfig(BaseModel):
+    """
+    Chroma configs
+    """
+
+    CHROMA_HOST: Optional[str] = Field(
+        description='Chroma host',
+        default=None,
+    )
+
+    CHROMA_PORT: PositiveInt = Field(
+        description='Chroma port',
+        default=8000,
+    )
+
+    CHROMA_TENANT: Optional[str] = Field(
+        description='Chroma database',
+        default=None,
+    )
+
+    CHROMA_DATABASE: Optional[str] = Field(
+        description='Chroma database',
+        default=None,
+    )
+
+    CHROMA_AUTH_PROVIDER: Optional[str] = Field(
+        description='Chroma authentication provider',
+        default=None,
+    )
+
+    CHROMA_AUTH_CREDENTIALS: Optional[str] = Field(
+        description='Chroma authentication credentials',
+        default=None,
+    )

api/configs/middleware/vdb/milvus_config.py

@@ -0,0 +1,39 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class MilvusConfig(BaseModel):
+    """
+    Milvus configs
+    """
+
+    MILVUS_HOST: Optional[str] = Field(
+        description='Milvus host',
+        default=None,
+    )
+
+    MILVUS_PORT: PositiveInt = Field(
+        description='Milvus RestFul API port',
+        default=9091,
+    )
+
+    MILVUS_USER: Optional[str] = Field(
+        description='Milvus user',
+        default=None,
+    )
+
+    MILVUS_PASSWORD: Optional[str] = Field(
+        description='Milvus password',
+        default=None,
+    )
+
+    MILVUS_SECURE: bool = Field(
+        description='whether to use SSL connection for Milvus',
+        default=False,
+    )
+
+    MILVUS_DATABASE: str = Field(
+        description='Milvus database, default to `default`',
+        default='default',
+    )

api/configs/middleware/vdb/opensearch_config.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class OpenSearchConfig(BaseModel):
+    """
+    OpenSearch configs
+    """
+
+    OPENSEARCH_HOST: Optional[str] = Field(
+        description='OpenSearch host',
+        default=None,
+    )
+
+    OPENSEARCH_PORT: PositiveInt = Field(
+        description='OpenSearch port',
+        default=9200,
+    )
+
+    OPENSEARCH_USER: Optional[str] = Field(
+        description='OpenSearch user',
+        default=None,
+    )
+
+    OPENSEARCH_PASSWORD: Optional[str] = Field(
+        description='OpenSearch password',
+        default=None,
+    )
+
+    OPENSEARCH_SECURE: bool = Field(
+        description='whether to use SSL connection for OpenSearch',
+        default=False,
+    )

api/configs/middleware/vdb/oracle_config.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class OracleConfig(BaseModel):
+    """
+    ORACLE configs
+    """
+
+    ORACLE_HOST: Optional[str] = Field(
+        description='ORACLE host',
+        default=None,
+    )
+
+    ORACLE_PORT: Optional[PositiveInt] = Field(
+        description='ORACLE port',
+        default=1521,
+    )
+
+    ORACLE_USER: Optional[str] = Field(
+        description='ORACLE user',
+        default=None,
+    )
+
+    ORACLE_PASSWORD: Optional[str] = Field(
+        description='ORACLE password',
+        default=None,
+    )
+
+    ORACLE_DATABASE: Optional[str] = Field(
+        description='ORACLE database',
+        default=None,
+    )

api/configs/middleware/vdb/pgvector_config.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class PGVectorConfig(BaseModel):
+    """
+    PGVector configs
+    """
+
+    PGVECTOR_HOST: Optional[str] = Field(
+        description='PGVector host',
+        default=None,
+    )
+
+    PGVECTOR_PORT: Optional[PositiveInt] = Field(
+        description='PGVector port',
+        default=5433,
+    )
+
+    PGVECTOR_USER: Optional[str] = Field(
+        description='PGVector user',
+        default=None,
+    )
+
+    PGVECTOR_PASSWORD: Optional[str] = Field(
+        description='PGVector password',
+        default=None,
+    )
+
+    PGVECTOR_DATABASE: Optional[str] = Field(
+        description='PGVector database',
+        default=None,
+    )

api/configs/middleware/vdb/pgvectors_config.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class PGVectoRSConfig(BaseModel):
+    """
+    PGVectoRS configs
+    """
+
+    PGVECTO_RS_HOST: Optional[str] = Field(
+        description='PGVectoRS host',
+        default=None,
+    )
+
+    PGVECTO_RS_PORT: Optional[PositiveInt] = Field(
+        description='PGVectoRS port',
+        default=5431,
+    )
+
+    PGVECTO_RS_USER: Optional[str] = Field(
+        description='PGVectoRS user',
+        default=None,
+    )
+
+    PGVECTO_RS_PASSWORD: Optional[str] = Field(
+        description='PGVectoRS password',
+        default=None,
+    )
+
+    PGVECTO_RS_DATABASE: Optional[str] = Field(
+        description='PGVectoRS database',
+        default=None,
+    )

api/configs/middleware/vdb/qdrant_config.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, NonNegativeInt, PositiveInt
+
+
+class QdrantConfig(BaseModel):
+    """
+    Qdrant configs
+    """
+
+    QDRANT_URL: Optional[str] = Field(
+        description='Qdrant url',
+        default=None,
+    )
+
+    QDRANT_API_KEY: Optional[str] = Field(
+        description='Qdrant api key',
+        default=None,
+    )
+
+    QDRANT_CLIENT_TIMEOUT: NonNegativeInt = Field(
+        description='Qdrant client timeout in seconds',
+        default=20,
+    )
+
+    QDRANT_GRPC_ENABLED: bool = Field(
+        description='whether enable grpc support for Qdrant connection',
+        default=False,
+    )
+
+    QDRANT_GRPC_PORT: PositiveInt = Field(
+        description='Qdrant grpc port',
+        default=6334,
+    )

api/configs/middleware/vdb/relyt_config.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class RelytConfig(BaseModel):
+    """
+    Relyt configs
+    """
+
+    RELYT_HOST: Optional[str] = Field(
+        description='Relyt host',
+        default=None,
+    )
+
+    RELYT_PORT: PositiveInt = Field(
+        description='Relyt port',
+        default=9200,
+    )
+
+    RELYT_USER: Optional[str] = Field(
+        description='Relyt user',
+        default=None,
+    )
+
+    RELYT_PASSWORD: Optional[str] = Field(
+        description='Relyt password',
+        default=None,
+    )
+
+    RELYT_DATABASE: Optional[str] = Field(
+        description='Relyt database',
+        default='default',
+    )

api/configs/middleware/vdb/tencent_vector_config.py

@@ -0,0 +1,44 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class TencentVectorDBConfig(BaseModel):
+    """
+    Tencent Vector configs
+    """
+
+    TENCENT_VECTOR_DB_URL: Optional[str] = Field(
+        description='Tencent Vector URL',
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_API_KEY: Optional[str] = Field(
+        description='Tencent Vector API key',
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_TIMEOUT: PositiveInt = Field(
+        description='Tencent Vector timeout in seconds',
+        default=30,
+    )
+
+    TENCENT_VECTOR_DB_USERNAME: Optional[str] = Field(
+        description='Tencent Vector password',
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_PASSWORD: Optional[str] = Field(
+        description='Tencent Vector password',
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_SHARD: PositiveInt = Field(
+        description='Tencent Vector sharding number',
+        default=1,
+    )
+
+    TENCENT_VECTOR_DB_REPLICAS: PositiveInt = Field(
+        description='Tencent Vector replicas',
+        default=2,
+    )

api/configs/middleware/vdb/tidb_vector_config.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class TiDBVectorConfig(BaseModel):
+    """
+    TiDB Vector configs
+    """
+
+    TIDB_VECTOR_HOST: Optional[str] = Field(
+        description='TiDB Vector host',
+        default=None,
+    )
+
+    TIDB_VECTOR_PORT: Optional[PositiveInt] = Field(
+        description='TiDB Vector port',
+        default=4000,
+    )
+
+    TIDB_VECTOR_USER: Optional[str] = Field(
+        description='TiDB Vector user',
+        default=None,
+    )
+
+    TIDB_VECTOR_PASSWORD: Optional[str] = Field(
+        description='TiDB Vector password',
+        default=None,
+    )
+
+    TIDB_VECTOR_DATABASE: Optional[str] = Field(
+        description='TiDB Vector database',
+        default=None,
+    )

api/configs/middleware/vdb/weaviate_config.py

@@ -0,0 +1,29 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class WeaviateConfig(BaseModel):
+    """
+    Weaviate configs
+    """
+
+    WEAVIATE_ENDPOINT: Optional[str] = Field(
+        description='Weaviate endpoint URL',
+        default=None,
+    )
+
+    WEAVIATE_API_KEY: Optional[str] = Field(
+        description='Weaviate API key',
+        default=None,
+    )
+
+    WEAVIATE_GRPC_ENABLED: bool = Field(
+        description='whether to enable gRPC for Weaviate connection',
+        default=True,
+    )
+
+    WEAVIATE_BATCH_SIZE: PositiveInt = Field(
+        description='Weaviate batch size',
+        default=100,
+    )

api/configs/packaging/__init__.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel, Field
+
+
+class PackagingInfo(BaseModel):
+    """
+    Packaging build information
+    """
+
+    CURRENT_VERSION: str = Field(
+        description='Dify version',
+        default='0.6.12',
+    )
+
+    COMMIT_SHA: str = Field(
+        description="SHA-1 checksum of the git commit used to build the app",
+        default='',
+    )

api/constants/languages.py

@@ -1,7 +1,3 @@
-
-
-languages = ['en-US', 'zh-Hans', 'zh-Hant', 'pt-BR', 'es-ES', 'fr-FR', 'de-DE', 'ja-JP', 'ko-KR', 'ru-RU', 'it-IT', 'uk-UA', 'vi-VN', 'pl-PL']
-
 language_timezone_mapping = {
     'en-US': 'America/New_York',
     'zh-Hans': 'Asia/Shanghai',
@@ -18,8 +14,11 @@ language_timezone_mapping = {
     'vi-VN': 'Asia/Ho_Chi_Minh',
     'ro-RO': 'Europe/Bucharest',
     'pl-PL': 'Europe/Warsaw',
+    'hi-IN': 'Asia/Kolkata'
 }
 
+languages = list(language_timezone_mapping.keys())
+
 
 def supported_language(lang):
     if lang in languages:

api/constants/model_template.py

@@ -22,7 +22,7 @@ default_app_templates = {
         'model_config': {
             'model': {
                 "provider": "openai",
-                "name": "gpt-4",
+                "name": "gpt-4o",
                 "mode": "chat",
                 "completion_params": {}
             },
@@ -51,7 +51,7 @@ default_app_templates = {
         'model_config': {
             'model': {
                 "provider": "openai",
-                "name": "gpt-4",
+                "name": "gpt-4o",
                 "mode": "chat",
                 "completion_params": {}
             }
@@ -77,7 +77,7 @@ default_app_templates = {
         'model_config': {
             'model': {
                 "provider": "openai",
-                "name": "gpt-4",
+                "name": "gpt-4o",
                 "mode": "chat",
                 "completion_params": {}
             }

api/controllers/console/__init__.py

@@ -20,6 +20,7 @@ from .app import (
     generator,
     message,
     model_config,
+    ops_trace,
     site,
     statistic,
     workflow,

api/controllers/console/app/app.py

@@ -1,4 +1,3 @@
-import json
 import uuid
 
 from flask_login import current_user
@@ -9,17 +8,14 @@ from controllers.console import api
 from controllers.console.app.wraps import get_app_model
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
-from core.tools.tool_manager import ToolManager
-from core.tools.utils.configuration import ToolParameterConfigurationManager
+from core.ops.ops_trace_manager import OpsTraceManager
 from fields.app_fields import (
     app_detail_fields,
     app_detail_fields_with_site,
     app_pagination_fields,
 )
 from libs.login import login_required
-from models.model import App, AppMode, AppModelConfig
 from services.app_service import AppService
-from services.tag_service import TagService
 
 ALLOW_CREATE_APP_MODES = ['chat', 'agent-chat', 'advanced-chat', 'workflow', 'completion']
 
@@ -129,6 +125,10 @@ class AppApi(Resource):
     @marshal_with(app_detail_fields_with_site)
     def put(self, app_model):
         """Update app"""
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('name', type=str, required=True, nullable=False, location='json')
         parser.add_argument('description', type=str, location='json')
@@ -147,6 +147,7 @@ class AppApi(Resource):
     @get_app_model
     def delete(self, app_model):
         """Delete app"""
+        # The role of the current user in the ta table must be admin, owner, or editor
         if not current_user.is_editor:
             raise Forbidden()
 
@@ -203,6 +204,10 @@ class AppNameApi(Resource):
     @get_app_model
     @marshal_with(app_detail_fields)
     def post(self, app_model):
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('name', type=str, required=True, location='json')
         args = parser.parse_args()
@@ -220,6 +225,10 @@ class AppIconApi(Resource):
     @get_app_model
     @marshal_with(app_detail_fields)
     def post(self, app_model):
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('icon', type=str, location='json')
         parser.add_argument('icon_background', type=str, location='json')
@@ -241,6 +250,7 @@ class AppSiteStatus(Resource):
         # The role of the current user in the ta table must be admin, owner, or editor
         if not current_user.is_editor:
             raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('enable_site', type=bool, required=True, location='json')
         args = parser.parse_args()
@@ -261,6 +271,7 @@ class AppApiStatus(Resource):
         # The role of the current user in the ta table must be admin or owner
         if not current_user.is_admin_or_owner:
             raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('enable_api', type=bool, required=True, location='json')
         args = parser.parse_args()
@@ -271,6 +282,39 @@ class AppApiStatus(Resource):
         return app_model
 
 
+class AppTraceApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, app_id):
+        """Get app trace"""
+        app_trace_config = OpsTraceManager.get_app_tracing_config(
+            app_id=app_id
+        )
+
+        return app_trace_config
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id):
+        # add app trace
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
+        parser = reqparse.RequestParser()
+        parser.add_argument('enabled', type=bool, required=True, location='json')
+        parser.add_argument('tracing_provider', type=str, required=True, location='json')
+        args = parser.parse_args()
+
+        OpsTraceManager.update_app_tracing_config(
+            app_id=app_id,
+            enabled=args['enabled'],
+            tracing_provider=args['tracing_provider'],
+        )
+
+        return {"result": "success"}
+
+
 api.add_resource(AppListApi, '/apps')
 api.add_resource(AppImportApi, '/apps/import')
 api.add_resource(AppApi, '/apps/<uuid:app_id>')
@@ -280,3 +324,4 @@ api.add_resource(AppNameApi, '/apps/<uuid:app_id>/name')
 api.add_resource(AppIconApi, '/apps/<uuid:app_id>/icon')
 api.add_resource(AppSiteStatus, '/apps/<uuid:app_id>/site-enable')
 api.add_resource(AppApiStatus, '/apps/<uuid:app_id>/api-enable')
+api.add_resource(AppTraceApi, '/apps/<uuid:app_id>/trace')

api/controllers/console/app/error.py

@@ -97,3 +97,21 @@ class DraftWorkflowNotSync(BaseHTTPException):
     error_code = 'draft_workflow_not_sync'
     description = "Workflow graph might have been modified, please refresh and resubmit."
     code = 400
+
+
+class TracingConfigNotExist(BaseHTTPException):
+    error_code = 'trace_config_not_exist'
+    description = "Trace config not exist."
+    code = 400
+
+
+class TracingConfigIsExist(BaseHTTPException):
+    error_code = 'trace_config_is_exist'
+    description = "Trace config is exist."
+    code = 400
+
+
+class TracingConfigCheckError(BaseHTTPException):
+    error_code = 'trace_config_check_error'
+    description = "Invalid Credentials."
+    code = 400

api/controllers/console/app/model_config.py

@@ -25,6 +25,7 @@ class ModelConfigResource(Resource):
     @account_initialization_required
     @get_app_model(mode=[AppMode.AGENT_CHAT, AppMode.CHAT, AppMode.COMPLETION])
     def post(self, app_model):
+        
         """Modify app model config"""
         # validate config
         model_configuration = AppModelConfigService.validate_configuration(

api/controllers/console/app/ops_trace.py

@@ -0,0 +1,101 @@
+from flask_restful import Resource, reqparse
+
+from controllers.console import api
+from controllers.console.app.error import TracingConfigCheckError, TracingConfigIsExist, TracingConfigNotExist
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from libs.login import login_required
+from services.ops_service import OpsService
+
+
+class TraceAppConfigApi(Resource):
+    """
+    Manage trace app configurations
+    """
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, app_id):
+        parser = reqparse.RequestParser()
+        parser.add_argument('tracing_provider', type=str, required=True, location='args')
+        args = parser.parse_args()
+
+        try:
+            trace_config = OpsService.get_tracing_app_config(
+                app_id=app_id, tracing_provider=args['tracing_provider']
+                )
+            if not trace_config:
+                return {"has_not_configured": True}
+            return trace_config
+        except Exception as e:
+            raise e
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id):
+        """Create a new trace app configuration"""
+        parser = reqparse.RequestParser()
+        parser.add_argument('tracing_provider', type=str, required=True, location='json')
+        parser.add_argument('tracing_config', type=dict, required=True, location='json')
+        args = parser.parse_args()
+
+        try:
+            result = OpsService.create_tracing_app_config(
+                app_id=app_id,
+                tracing_provider=args['tracing_provider'],
+                tracing_config=args['tracing_config']
+            )
+            if not result:
+                raise TracingConfigIsExist()
+            if result.get('error'):
+                raise TracingConfigCheckError()
+            return result
+        except Exception as e:
+            raise e
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def patch(self, app_id):
+        """Update an existing trace app configuration"""
+        parser = reqparse.RequestParser()
+        parser.add_argument('tracing_provider', type=str, required=True, location='json')
+        parser.add_argument('tracing_config', type=dict, required=True, location='json')
+        args = parser.parse_args()
+
+        try:
+            result = OpsService.update_tracing_app_config(
+                app_id=app_id,
+                tracing_provider=args['tracing_provider'],
+                tracing_config=args['tracing_config']
+            )
+            if not result:
+                raise TracingConfigNotExist()
+            return {"result": "success"}
+        except Exception as e:
+            raise e
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, app_id):
+        """Delete an existing trace app configuration"""
+        parser = reqparse.RequestParser()
+        parser.add_argument('tracing_provider', type=str, required=True, location='args')
+        args = parser.parse_args()
+
+        try:
+            result = OpsService.delete_tracing_app_config(
+                app_id=app_id,
+                tracing_provider=args['tracing_provider']
+            )
+            if not result:
+                raise TracingConfigNotExist()
+            return {"result": "success"}
+        except Exception as e:
+            raise e
+
+
+api.add_resource(TraceAppConfigApi, '/apps/<uuid:app_id>/trace-config')

api/controllers/console/app/site.py

@@ -20,6 +20,8 @@ def parse_app_site_args():
     parser.add_argument('icon_background', type=str, required=False, location='json')
     parser.add_argument('description', type=str, required=False, location='json')
     parser.add_argument('default_language', type=supported_language, required=False, location='json')
+    parser.add_argument('chat_color_theme', type=str, required=False, location='json')
+    parser.add_argument('chat_color_theme_inverted', type=bool, required=False, location='json')
     parser.add_argument('customize_domain', type=str, required=False, location='json')
     parser.add_argument('copyright', type=str, required=False, location='json')
     parser.add_argument('privacy_policy', type=str, required=False, location='json')
@@ -28,6 +30,7 @@ def parse_app_site_args():
                         required=False,
                         location='json')
     parser.add_argument('prompt_public', type=bool, required=False, location='json')
+    parser.add_argument('show_workflow_steps', type=bool, required=False, location='json')
     return parser.parse_args()
 
 
@@ -54,12 +57,15 @@ class AppSite(Resource):
             'icon_background',
             'description',
             'default_language',
+            'chat_color_theme',
+            'chat_color_theme_inverted',
             'customize_domain',
             'copyright',
             'privacy_policy',
             'custom_disclaimer',
             'customize_token_strategy',
-            'prompt_public'
+            'prompt_public',
+            'show_workflow_steps'
         ]:
             value = args.get(attr_name)
             if value is not None:

api/controllers/console/app/workflow.py

@@ -3,7 +3,7 @@ import logging
 
 from flask import abort, request
 from flask_restful import Resource, marshal_with, reqparse
-from werkzeug.exceptions import InternalServerError, NotFound
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 import services
 from controllers.console import api
@@ -36,6 +36,10 @@ class DraftWorkflowApi(Resource):
         """
         Get draft workflow
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         # fetch draft workflow by app_model
         workflow_service = WorkflowService()
         workflow = workflow_service.get_draft_workflow(app_model=app_model)
@@ -54,6 +58,10 @@ class DraftWorkflowApi(Resource):
         """
         Sync draft workflow
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         content_type = request.headers.get('Content-Type')
 
         if 'application/json' in content_type:
@@ -101,6 +109,34 @@ class DraftWorkflowApi(Resource):
         }
 
 
+class DraftWorkflowImportApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    @marshal_with(workflow_fields)
+    def post(self, app_model: App):
+        """
+        Import draft workflow
+        """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('data', type=str, required=True, nullable=False, location='json')
+        args = parser.parse_args()
+
+        workflow_service = WorkflowService()
+        workflow = workflow_service.import_draft_workflow(
+            app_model=app_model,
+            data=args['data'],
+            account=current_user
+        )
+
+        return workflow
+
+
 class AdvancedChatDraftWorkflowRunApi(Resource):
     @setup_required
     @login_required
@@ -110,6 +146,10 @@ class AdvancedChatDraftWorkflowRunApi(Resource):
         """
         Run draft workflow
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, location='json')
         parser.add_argument('query', type=str, required=True, location='json', default='')
@@ -146,6 +186,10 @@ class AdvancedChatDraftRunIterationNodeApi(Resource):
         """
         Run draft workflow iteration node
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, location='json')
         args = parser.parse_args()
@@ -179,6 +223,10 @@ class WorkflowDraftRunIterationNodeApi(Resource):
         """
         Run draft workflow iteration node
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, location='json')
         args = parser.parse_args()
@@ -212,6 +260,10 @@ class DraftWorkflowRunApi(Resource):
         """
         Run draft workflow
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, required=True, nullable=False, location='json')
         parser.add_argument('files', type=list, required=False, location='json')
@@ -243,6 +295,10 @@ class WorkflowTaskStopApi(Resource):
         """
         Stop workflow task
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         AppQueueManager.set_stop_flag(task_id, InvokeFrom.DEBUGGER, current_user.id)
 
         return {
@@ -260,6 +316,10 @@ class DraftWorkflowNodeRunApi(Resource):
         """
         Run draft workflow node
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, required=True, nullable=False, location='json')
         args = parser.parse_args()
@@ -286,6 +346,10 @@ class PublishedWorkflowApi(Resource):
         """
         Get published workflow
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         # fetch published workflow by app_model
         workflow_service = WorkflowService()
         workflow = workflow_service.get_published_workflow(app_model=app_model)
@@ -301,6 +365,10 @@ class PublishedWorkflowApi(Resource):
         """
         Publish workflow
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         workflow_service = WorkflowService()
         workflow = workflow_service.publish_workflow(app_model=app_model, account=current_user)
 
@@ -319,6 +387,10 @@ class DefaultBlockConfigsApi(Resource):
         """
         Get default block config
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         # Get default block configs
         workflow_service = WorkflowService()
         return workflow_service.get_default_block_configs()
@@ -333,6 +405,10 @@ class DefaultBlockConfigApi(Resource):
         """
         Get default block config
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('q', type=str, location='args')
         args = parser.parse_args()
@@ -363,6 +439,10 @@ class ConvertToWorkflowApi(Resource):
         Convert expert mode of chatbot app to workflow mode
         Convert Completion App to Workflow App
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         if request.data:
             parser = reqparse.RequestParser()
             parser.add_argument('name', type=str, required=False, nullable=True, location='json')
@@ -387,6 +467,7 @@ class ConvertToWorkflowApi(Resource):
 
 
 api.add_resource(DraftWorkflowApi, '/apps/<uuid:app_id>/workflows/draft')
+api.add_resource(DraftWorkflowImportApi, '/apps/<uuid:app_id>/workflows/draft/import')
 api.add_resource(AdvancedChatDraftWorkflowRunApi, '/apps/<uuid:app_id>/advanced-chat/workflows/draft/run')
 api.add_resource(DraftWorkflowRunApi, '/apps/<uuid:app_id>/workflows/draft/run')
 api.add_resource(WorkflowTaskStopApi, '/apps/<uuid:app_id>/workflow-runs/tasks/<string:task_id>/stop')

api/controllers/console/auth/data_source_bearer_auth.py

@@ -16,15 +16,21 @@ class ApiKeyAuthDataSource(Resource):
     @login_required
     @account_initialization_required
     def get(self):
-        # The role of the current user in the table must be admin or owner
-        if not current_user.is_admin_or_owner:
-            raise Forbidden()
         data_source_api_key_bindings = ApiKeyAuthService.get_provider_auth_list(current_user.current_tenant_id)
         if data_source_api_key_bindings:
             return {
-                'settings': [data_source_api_key_binding.to_dict() for data_source_api_key_binding in
-                             data_source_api_key_bindings]}
-        return {'settings': []}
+                'sources': [{
+                                'id': data_source_api_key_binding.id,
+                                'category': data_source_api_key_binding.category,
+                                'provider': data_source_api_key_binding.provider,
+                                'disabled': data_source_api_key_binding.disabled,
+                                'created_at': int(data_source_api_key_binding.created_at.timestamp()),
+                                'updated_at': int(data_source_api_key_binding.updated_at.timestamp()),
+                            }
+                            for data_source_api_key_binding in
+                             data_source_api_key_bindings]
+            }
+        return {'sources': []}
 
 
 class ApiKeyAuthDataSourceBinding(Resource):

api/controllers/console/auth/login.py

@@ -1,3 +1,5 @@
+from typing import cast
+
 import flask_login
 from flask import current_app, request
 from flask_restful import Resource, reqparse
@@ -5,8 +7,9 @@ from flask_restful import Resource, reqparse
 import services
 from controllers.console import api
 from controllers.console.setup import setup_required
-from libs.helper import email
+from libs.helper import email, get_remote_ip
 from libs.password import valid_password
+from models.account import Account
 from services.account_service import AccountService, TenantService
 
 
@@ -34,10 +37,7 @@ class LoginApi(Resource):
         if len(tenants) == 0:
             return {'result': 'fail', 'data': 'workspace not found, please contact system admin to invite you to join in a workspace'}
 
-        AccountService.update_last_login(account, request)
-
-        # todo: return the user info
-        token = AccountService.get_account_jwt_token(account)
+        token = AccountService.login(account, ip_address=get_remote_ip(request))
 
         return {'result': 'success', 'data': token}
 
@@ -46,6 +46,9 @@ class LogoutApi(Resource):
 
     @setup_required
     def get(self):
+        account = cast(Account, flask_login.current_user)
+        token = request.headers.get('Authorization', '').split(' ')[1]
+        AccountService.logout(account=account, token=token)
         flask_login.logout_user()
         return {'result': 'success'}
 

api/controllers/console/auth/oauth.py

@@ -8,6 +8,7 @@ from flask_restful import Resource
 
 from constants.languages import languages
 from extensions.ext_database import db
+from libs.helper import get_remote_ip
 from libs.oauth import GitHubOAuth, GoogleOAuth, OAuthUserInfo
 from models.account import Account, AccountStatus
 from services.account_service import AccountService, RegisterService, TenantService
@@ -78,9 +79,7 @@ class OAuthCallback(Resource):
 
         TenantService.create_owner_tenant_if_not_exist(account)
 
-        AccountService.update_last_login(account, request)
-
-        token = AccountService.get_account_jwt_token(account)
+        token = AccountService.login(account, ip_address=get_remote_ip(request))
 
         return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?console_token={token}')
 

api/controllers/console/datasets/datasets.py

@@ -8,7 +8,7 @@ import services
 from controllers.console import api
 from controllers.console.apikey import api_key_fields, api_key_list
 from controllers.console.app.error import ProviderNotInitializeError
-from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError
+from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
@@ -17,6 +17,7 @@ from core.model_runtime.entities.model_entities import ModelType
 from core.provider_manager import ProviderManager
 from core.rag.datasource.vdb.vector_type import VectorType
 from core.rag.extractor.entity.extract_setting import ExtractSetting
+from core.rag.retrieval.retrival_methods import RetrievalMethod
 from extensions.ext_database import db
 from fields.app_fields import related_app_list
 from fields.dataset_fields import dataset_detail_fields, dataset_query_detail_fields
@@ -345,6 +346,8 @@ class DatasetIndexingEstimateApi(Resource):
                 "in the Settings -> Model Provider.")
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
+        except Exception as e:
+            raise IndexingEstimateError(str(e))
 
         return response, 200
 
@@ -497,16 +500,18 @@ class DatasetRetrievalSettingApi(Resource):
     def get(self):
         vector_type = current_app.config['VECTOR_STORE']
         match vector_type:
-            case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT:
+            case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE:
                 return {
                     'retrieval_method': [
-                        'semantic_search'
+                        RetrievalMethod.SEMANTIC_SEARCH
                     ]
                 }
-            case VectorType.QDRANT | VectorType.WEAVIATE:
+            case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH:
                 return {
                     'retrieval_method': [
-                        'semantic_search', 'full_text_search', 'hybrid_search'
+                        RetrievalMethod.SEMANTIC_SEARCH,
+                        RetrievalMethod.FULL_TEXT_SEARCH,
+                        RetrievalMethod.HYBRID_SEARCH,
                     ]
                 }
             case _:
@@ -519,16 +524,18 @@ class DatasetRetrievalSettingMockApi(Resource):
     @account_initialization_required
     def get(self, vector_type):
         match vector_type:
-            case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCEN:
+            case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE:
                 return {
                     'retrieval_method': [
-                        'semantic_search'
+                        RetrievalMethod.SEMANTIC_SEARCH
                     ]
                 }
-            case VectorType.QDRANT | VectorType.WEAVIATE:
+            case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH:
                 return {
                     'retrieval_method': [
-                        'semantic_search', 'full_text_search', 'hybrid_search'
+                        RetrievalMethod.SEMANTIC_SEARCH,
+                        RetrievalMethod.FULL_TEXT_SEARCH,
+                        RetrievalMethod.HYBRID_SEARCH,
                     ]
                 }
             case _:

api/controllers/console/datasets/datasets_document.py

@@ -20,6 +20,7 @@ from controllers.console.datasets.error import (
     ArchivedDocumentImmutableError,
     DocumentAlreadyFinishedError,
     DocumentIndexingError,
+    IndexingEstimateError,
     InvalidActionError,
     InvalidMetadataError,
 )
@@ -388,6 +389,8 @@ class DocumentIndexingEstimateApi(DocumentResource):
                         "in the Settings -> Model Provider.")
                 except ProviderTokenNotInitError as ex:
                     raise ProviderNotInitializeError(ex.description)
+                except Exception as e:
+                    raise IndexingEstimateError(str(e))
 
         return response
 
@@ -493,6 +496,8 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
                     "in the Settings -> Model Provider.")
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
+            except Exception as e:
+                raise IndexingEstimateError(str(e))
         return response
 
 

api/controllers/console/datasets/error.py

@@ -83,3 +83,9 @@ class DatasetInUseError(BaseHTTPException):
     error_code = 'dataset_in_use'
     description = "The dataset is being used by some apps. Please remove the dataset from the apps before deleting it."
     code = 409
+
+
+class IndexingEstimateError(BaseHTTPException):
+    error_code = 'indexing_estimate_error'
+    description = "Knowledge indexing estimate failed: {message}"
+    code = 500

api/controllers/console/datasets/file.py

@@ -14,7 +14,7 @@ from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from fields.file_fields import file_fields, upload_config_fields
 from libs.login import login_required
-from services.file_service import ALLOWED_EXTENSIONS, UNSTRUSTURED_ALLOWED_EXTENSIONS, FileService
+from services.file_service import ALLOWED_EXTENSIONS, UNSTRUCTURED_ALLOWED_EXTENSIONS, FileService
 
 PREVIEW_WORDS_LIMIT = 3000
 
@@ -77,7 +77,7 @@ class FileSupportTypeApi(Resource):
     @account_initialization_required
     def get(self):
         etl_type = current_app.config['ETL_TYPE']
-        allowed_extensions = UNSTRUSTURED_ALLOWED_EXTENSIONS if etl_type == 'Unstructured' else ALLOWED_EXTENSIONS
+        allowed_extensions = UNSTRUCTURED_ALLOWED_EXTENSIONS if etl_type == 'Unstructured' else ALLOWED_EXTENSIONS
         return {'allowed_extensions': allowed_extensions}
 
 

api/controllers/console/setup.py

@@ -4,7 +4,7 @@ from flask import current_app, request
 from flask_restful import Resource, reqparse
 
 from extensions.ext_database import db
-from libs.helper import email, str_len
+from libs.helper import email, get_remote_ip, str_len
 from libs.password import valid_password
 from models.model import DifySetup
 from services.account_service import AccountService, RegisterService, TenantService
@@ -61,7 +61,7 @@ class SetupApi(Resource):
         TenantService.create_owner_tenant_if_not_exist(account)
 
         setup()
-        AccountService.update_last_login(account, request)
+        AccountService.update_last_login(account, ip_address=get_remote_ip(request))
 
         return {'result': 'success'}, 201
 

api/controllers/console/workspace/tool_providers.py

@@ -104,7 +104,7 @@ class ToolBuiltinProviderIconApi(Resource):
     @setup_required
     def get(self, provider):
         icon_bytes, mimetype = BuiltinToolManageService.get_builtin_tool_provider_icon(provider)
-        icon_cache_max_age = int(current_app.config.get('TOOL_ICON_CACHE_MAX_AGE'))
+        icon_cache_max_age = current_app.config.get('TOOL_ICON_CACHE_MAX_AGE')
         return send_file(io.BytesIO(icon_bytes), mimetype=mimetype, max_age=icon_cache_max_age)
 
 class ToolApiProviderAddApi(Resource):

api/controllers/web/site.py

@@ -26,6 +26,8 @@ class AppSiteApi(WebApiResource):
 
     site_fields = {
         'title': fields.String,
+        'chat_color_theme': fields.String,
+        'chat_color_theme_inverted': fields.Boolean,
         'icon': fields.String,
         'icon_background': fields.String,
         'description': fields.String,
@@ -33,7 +35,8 @@ class AppSiteApi(WebApiResource):
         'privacy_policy': fields.String,
         'custom_disclaimer': fields.String,
         'default_language': fields.String,
-        'prompt_public': fields.Boolean
+        'prompt_public': fields.Boolean,
+        'show_workflow_steps': fields.Boolean,
     }
 
     app_fields = {

api/core/agent/base_agent_runner.py

@@ -32,7 +32,6 @@ from core.model_runtime.entities.model_entities import ModelFeature
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.model_runtime.utils.encoders import jsonable_encoder
 from core.tools.entities.tool_entities import (
-    ToolInvokeMessage,
     ToolParameter,
     ToolRuntimeVariablePool,
 )
@@ -141,24 +140,6 @@ class BaseAgentRunner(AppRunner):
             app_generate_entity.app_config.prompt_template.simple_prompt_template = ''
 
         return app_generate_entity
-
-    def _convert_tool_response_to_str(self, tool_response: list[ToolInvokeMessage]) -> str:
-        """
-        Handle tool response
-        """
-        result = ''
-        for response in tool_response:
-            if response.type == ToolInvokeMessage.MessageType.TEXT:
-                result += response.message
-            elif response.type == ToolInvokeMessage.MessageType.LINK:
-                result += f"result link: {response.message}. please tell user to check it."
-            elif response.type == ToolInvokeMessage.MessageType.IMAGE_LINK or \
-                 response.type == ToolInvokeMessage.MessageType.IMAGE:
-                result += "image has been created and sent to user already, you do not need to create it, just tell the user to check it now."
-            else:
-                result += f"tool response: {response.message}."
-
-        return result
     
     def _convert_tool_to_prompt_message_tool(self, tool: AgentToolEntity) -> tuple[PromptMessageTool, Tool]:
         """

api/core/agent/cot_agent_runner.py

@@ -1,7 +1,7 @@
 import json
 from abc import ABC, abstractmethod
 from collections.abc import Generator
-from typing import Union
+from typing import Optional, Union
 
 from core.agent.base_agent_runner import BaseAgentRunner
 from core.agent.entities import AgentScratchpadUnit
@@ -15,6 +15,7 @@ from core.model_runtime.entities.message_entities import (
     ToolPromptMessage,
     UserPromptMessage,
 )
+from core.ops.ops_trace_manager import TraceQueueManager
 from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransform
 from core.tools.entities.tool_entities import ToolInvokeMeta
 from core.tools.tool.tool import Tool
@@ -32,9 +33,9 @@ class CotAgentRunner(BaseAgentRunner, ABC):
     _prompt_messages_tools: list[PromptMessage] = None
 
     def run(self, message: Message,
-        query: str,
-        inputs: dict[str, str],
-    ) -> Union[Generator, LLMResult]:
+            query: str,
+            inputs: dict[str, str],
+            ) -> Union[Generator, LLMResult]:
         """
         Run Cot agent application
         """
@@ -42,6 +43,8 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         self._repack_app_generate_entity(app_generate_entity)
         self._init_react_state(query)
 
+        trace_manager = app_generate_entity.trace_manager
+
         # check model mode
         if 'Observation' not in app_generate_entity.model_conf.stop:
             if app_generate_entity.model_conf.provider not in self._ignore_observation_providers:
@@ -52,7 +55,8 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         # init instruction
         inputs = inputs or {}
         instruction = app_config.prompt_template.simple_prompt_template
-        self._instruction = self._fill_in_inputs_from_external_data_tools(instruction, inputs)
+        self._instruction = self._fill_in_inputs_from_external_data_tools(
+            instruction, inputs)
 
         iteration_step = 1
         max_iteration_steps = min(app_config.agent.max_iteration, 5) + 1
@@ -60,8 +64,6 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         # convert tools into ModelRuntime Tool format
         tool_instances, self._prompt_messages_tools = self._init_prompt_tools()
 
-        prompt_messages = self._organize_prompt_messages()
-        
         function_call_state = True
         llm_usage = {
             'usage': None
@@ -120,9 +122,10 @@ class CotAgentRunner(BaseAgentRunner, ABC):
             # check llm result
             if not chunks:
                 raise ValueError("failed to invoke llm")
-            
+
             usage_dict = {}
-            react_chunks = CotAgentOutputParser.handle_react_stream_output(chunks, usage_dict)
+            react_chunks = CotAgentOutputParser.handle_react_stream_output(
+                chunks, usage_dict)
             scratchpad = AgentScratchpadUnit(
                 agent_response='',
                 thought='',
@@ -160,15 +163,16 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                         )
                     )
 
-            scratchpad.thought = scratchpad.thought.strip() or 'I am thinking about how to help you'
+            scratchpad.thought = scratchpad.thought.strip(
+            ) or 'I am thinking about how to help you'
             self._agent_scratchpad.append(scratchpad)
-            
+
             # get llm usage
             if 'usage' in usage_dict:
                 increase_usage(llm_usage, usage_dict['usage'])
             else:
                 usage_dict['usage'] = LLMUsage.empty_usage()
-            
+
             self.save_agent_thought(
                 agent_thought=agent_thought,
                 tool_name=scratchpad.action.action_name if scratchpad.action else '',
@@ -182,7 +186,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                 messages_ids=[],
                 llm_usage=usage_dict['usage']
             )
-            
+
             if not scratchpad.is_final():
                 self.queue_manager.publish(QueueAgentThoughtEvent(
                     agent_thought_id=agent_thought.id
@@ -196,7 +200,8 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                     # action is final answer, return final answer directly
                     try:
                         if isinstance(scratchpad.action.action_input, dict):
-                            final_answer = json.dumps(scratchpad.action.action_input)
+                            final_answer = json.dumps(
+                                scratchpad.action.action_input)
                         elif isinstance(scratchpad.action.action_input, str):
                             final_answer = scratchpad.action.action_input
                         else:
@@ -207,9 +212,10 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                     function_call_state = True
                     # action is tool call, invoke tool
                     tool_invoke_response, tool_invoke_meta = self._handle_invoke_action(
-                        action=scratchpad.action, 
+                        action=scratchpad.action,
                         tool_instances=tool_instances,
-                        message_file_ids=message_file_ids
+                        message_file_ids=message_file_ids,
+                        trace_manager=trace_manager,
                     )
                     scratchpad.observation = tool_invoke_response
                     scratchpad.agent_response = tool_invoke_response
@@ -217,10 +223,13 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                     self.save_agent_thought(
                         agent_thought=agent_thought,
                         tool_name=scratchpad.action.action_name,
-                        tool_input={scratchpad.action.action_name: scratchpad.action.action_input},
+                        tool_input={
+                            scratchpad.action.action_name: scratchpad.action.action_input},
                         thought=scratchpad.thought,
-                        observation={scratchpad.action.action_name: tool_invoke_response},
-                        tool_invoke_meta={scratchpad.action.action_name: tool_invoke_meta.to_dict()},
+                        observation={
+                            scratchpad.action.action_name: tool_invoke_response},
+                        tool_invoke_meta={
+                            scratchpad.action.action_name: tool_invoke_meta.to_dict()},
                         answer=scratchpad.agent_response,
                         messages_ids=message_file_ids,
                         llm_usage=usage_dict['usage']
@@ -251,12 +260,12 @@ class CotAgentRunner(BaseAgentRunner, ABC):
 
         # save agent thought
         self.save_agent_thought(
-            agent_thought=agent_thought, 
+            agent_thought=agent_thought,
             tool_name='',
             tool_input={},
             tool_invoke_meta={},
             thought=final_answer,
-            observation={}, 
+            observation={},
             answer=final_answer,
             messages_ids=[]
         )
@@ -273,9 +282,11 @@ class CotAgentRunner(BaseAgentRunner, ABC):
             system_fingerprint=''
         )), PublishFrom.APPLICATION_MANAGER)
 
-    def _handle_invoke_action(self, action: AgentScratchpadUnit.Action, 
+    def _handle_invoke_action(self, action: AgentScratchpadUnit.Action,
                               tool_instances: dict[str, Tool],
-                              message_file_ids: list[str]) -> tuple[str, ToolInvokeMeta]:
+                              message_file_ids: list[str],
+                              trace_manager: Optional[TraceQueueManager] = None
+                              ) -> tuple[str, ToolInvokeMeta]:
         """
         handle invoke action
         :param action: action
@@ -290,7 +301,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         if not tool_instance:
             answer = f"there is not a tool named {tool_call_name}"
             return answer, ToolInvokeMeta.error_instance(answer)
-        
+
         if isinstance(tool_call_args, str):
             try:
                 tool_call_args = json.loads(tool_call_args)
@@ -305,20 +316,22 @@ class CotAgentRunner(BaseAgentRunner, ABC):
             tenant_id=self.tenant_id,
             message=self.message,
             invoke_from=self.application_generate_entity.invoke_from,
-            agent_tool_callback=self.agent_callback
+            agent_tool_callback=self.agent_callback,
+            trace_manager=trace_manager,
         )
 
         # publish files
-        for message_file, save_as in message_files:
+        for message_file_id, save_as in message_files:
             if save_as:
-                self.variables_pool.set_file(tool_name=tool_call_name, value=message_file.id, name=save_as)
+                self.variables_pool.set_file(
+                    tool_name=tool_call_name, value=message_file_id, name=save_as)
 
             # publish message file
             self.queue_manager.publish(QueueMessageFileEvent(
-                message_file_id=message_file.id
+                message_file_id=message_file_id
             ), PublishFrom.APPLICATION_MANAGER)
             # add message file ids
-            message_file_ids.append(message_file.id)
+            message_file_ids.append(message_file_id)
 
         return tool_invoke_response, tool_invoke_meta
 
@@ -342,7 +355,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                 continue
 
         return instruction
-    
+
     def _init_react_state(self, query) -> None:
         """
         init agent scratchpad
@@ -350,7 +363,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         self._query = query
         self._agent_scratchpad = []
         self._historic_prompt_messages = self._organize_historic_prompt_messages()
-    
+
     @abstractmethod
     def _organize_prompt_messages(self) -> list[PromptMessage]:
         """
@@ -379,54 +392,54 @@ class CotAgentRunner(BaseAgentRunner, ABC):
             organize historic prompt messages
         """
         result: list[PromptMessage] = []
-        scratchpad: list[AgentScratchpadUnit] = []
+        scratchpads: list[AgentScratchpadUnit] = []
         current_scratchpad: AgentScratchpadUnit = None
 
-        self.history_prompt_messages = AgentHistoryPromptTransform(
-            model_config=self.model_config,
-            prompt_messages=current_session_messages or [],
-            history_messages=self.history_prompt_messages,
-            memory=self.memory
-        ).get_prompt()
-
         for message in self.history_prompt_messages:
             if isinstance(message, AssistantPromptMessage):
-                current_scratchpad = AgentScratchpadUnit(
-                    agent_response=message.content,
-                    thought=message.content or 'I am thinking about how to help you',
-                    action_str='',
-                    action=None,
-                    observation=None,
-                )
+                if not current_scratchpad:
+                    current_scratchpad = AgentScratchpadUnit(
+                        agent_response=message.content,
+                        thought=message.content or 'I am thinking about how to help you',
+                        action_str='',
+                        action=None,
+                        observation=None,
+                    )
+                    scratchpads.append(current_scratchpad)
                 if message.tool_calls:
                     try:
                         current_scratchpad.action = AgentScratchpadUnit.Action(
                             action_name=message.tool_calls[0].function.name,
-                            action_input=json.loads(message.tool_calls[0].function.arguments)
+                            action_input=json.loads(
+                                message.tool_calls[0].function.arguments)
                         )
                         current_scratchpad.action_str = json.dumps(
                             current_scratchpad.action.to_dict()
                         )
                     except:
                         pass
-                
-                scratchpad.append(current_scratchpad)
             elif isinstance(message, ToolPromptMessage):
                 if current_scratchpad:
                     current_scratchpad.observation = message.content
             elif isinstance(message, UserPromptMessage):
+                if scratchpads:
+                    result.append(AssistantPromptMessage(
+                        content=self._format_assistant_message(scratchpads)
+                    ))
+                    scratchpads = []
+                    current_scratchpad = None
+
                 result.append(message)
 
-                if scratchpad:
-                    result.append(AssistantPromptMessage(
-                        content=self._format_assistant_message(scratchpad)
-                    ))
-
-                scratchpad = []
-
-        if scratchpad:
+        if scratchpads:
             result.append(AssistantPromptMessage(
-                content=self._format_assistant_message(scratchpad)
+                content=self._format_assistant_message(scratchpads)
             ))
-        
-        return result
+
+        historic_prompts = AgentHistoryPromptTransform(
+            model_config=self.model_config,
+            prompt_messages=current_session_messages or [],
+            history_messages=result,
+            memory=self.memory
+        ).get_prompt()
+        return historic_prompts

api/core/agent/cot_chat_agent_runner.py

@@ -5,6 +5,7 @@ from core.model_runtime.entities.message_entities import (
     AssistantPromptMessage,
     PromptMessage,
     SystemPromptMessage,
+    TextPromptMessageContent,
     UserPromptMessage,
 )
 from core.model_runtime.utils.encoders import jsonable_encoder
@@ -25,6 +26,21 @@ class CotChatAgentRunner(CotAgentRunner):
 
         return SystemPromptMessage(content=system_prompt)
 
+    def _organize_user_query(self, query,  prompt_messages: list[PromptMessage] = None) -> list[PromptMessage]:
+        """
+        Organize user query
+        """
+        if self.files:
+            prompt_message_contents = [TextPromptMessageContent(data=query)]
+            for file_obj in self.files:
+                prompt_message_contents.append(file_obj.prompt_message_content)
+
+            prompt_messages.append(UserPromptMessage(content=prompt_message_contents))
+        else:
+            prompt_messages.append(UserPromptMessage(content=query))
+
+        return prompt_messages
+
     def _organize_prompt_messages(self) -> list[PromptMessage]:
         """
         Organize 
@@ -51,27 +67,27 @@ class CotChatAgentRunner(CotAgentRunner):
             assistant_messages = [assistant_message]
 
         # query messages
-        query_messages = UserPromptMessage(content=self._query)
+        query_messages = self._organize_user_query(self._query, [])
 
         if assistant_messages:
             # organize historic prompt messages
             historic_messages = self._organize_historic_prompt_messages([
                 system_message,
-                query_messages,
+                *query_messages,
                 *assistant_messages,
                 UserPromptMessage(content='continue')
-            ])            
+            ])
             messages = [
                 system_message,
                 *historic_messages,
-                query_messages,
+                *query_messages,
                 *assistant_messages,
                 UserPromptMessage(content='continue')
             ]
         else:
             # organize historic prompt messages
-            historic_messages = self._organize_historic_prompt_messages([system_message, query_messages])
-            messages = [system_message, *historic_messages, query_messages]
+            historic_messages = self._organize_historic_prompt_messages([system_message, *query_messages])
+            messages = [system_message, *historic_messages, *query_messages]
 
         # join all messages
-        return messages
+        return messages

api/core/agent/fc_agent_runner.py

@@ -50,6 +50,9 @@ class FunctionCallAgentRunner(BaseAgentRunner):
         }
         final_answer = ''
 
+        # get tracing instance
+        trace_manager = app_generate_entity.trace_manager
+        
         def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage):
             if not final_llm_usage_dict['usage']:
                 final_llm_usage_dict['usage'] = usage
@@ -243,18 +246,19 @@ class FunctionCallAgentRunner(BaseAgentRunner):
                         message=self.message,
                         invoke_from=self.application_generate_entity.invoke_from,
                         agent_tool_callback=self.agent_callback,
+                        trace_manager=trace_manager,
                     )
                     # publish files
-                    for message_file, save_as in message_files:
+                    for message_file_id, save_as in message_files:
                         if save_as:
-                            self.variables_pool.set_file(tool_name=tool_call_name, value=message_file.id, name=save_as)
+                            self.variables_pool.set_file(tool_name=tool_call_name, value=message_file_id, name=save_as)
 
                         # publish message file
                         self.queue_manager.publish(QueueMessageFileEvent(
-                            message_file_id=message_file.id
+                            message_file_id=message_file_id
                         ), PublishFrom.APPLICATION_MANAGER)
                         # add message file ids
-                        message_file_ids.append(message_file.id)
+                        message_file_ids.append(message_file_id)
                     
                     tool_response = {
                         "tool_call_id": tool_call_id,

api/core/agent/output_parser/cot_output_parser.py

@@ -17,6 +17,10 @@ class CotAgentOutputParser:
                 action_name = None
                 action_input = None
 
+                # cohere always returns a list
+                if isinstance(action, list) and len(action) == 1:
+                    action = action[0]
+
                 for key, value in action.items():
                     if 'input' in key.lower():
                         action_input = value

api/core/app/app_config/easy_ui_based_app/agent/manager.py

@@ -40,7 +40,7 @@ class AgentConfigManager:
                         'provider_type': tool['provider_type'],
                         'provider_id': tool['provider_id'],
                         'tool_name': tool['tool_name'],
-                        'tool_parameters': tool['tool_parameters'] if 'tool_parameters' in tool else {}
+                        'tool_parameters': tool.get('tool_parameters', {})
                     }
 
                     agent_tools.append(AgentToolEntity(**agent_tool_properties))

api/core/app/app_config/entities.py

@@ -183,6 +183,14 @@ class TextToSpeechEntity(BaseModel):
     language: Optional[str] = None
 
 
+class TracingConfigEntity(BaseModel):
+    """
+    Tracing Config Entity.
+    """
+    enabled: bool
+    tracing_provider: str
+
+
 class FileExtraConfig(BaseModel):
     """
     File Upload Entity.
@@ -199,7 +207,7 @@ class AppAdditionalFeatures(BaseModel):
     more_like_this: bool = False
     speech_to_text: bool = False
     text_to_speech: Optional[TextToSpeechEntity] = None
-
+    trace_config: Optional[TracingConfigEntity] = None
 
 class AppConfig(BaseModel):
     """

api/core/app/apps/advanced_chat/app_generator.py

@@ -20,6 +20,7 @@ from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity,
 from core.app.entities.task_entities import ChatbotAppBlockingResponse, ChatbotAppStreamResponse
 from core.file.message_file_parser import MessageFileParser
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
+from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
 from models.account import Account
 from models.model import App, Conversation, EndUser, Message
@@ -29,13 +30,14 @@ logger = logging.getLogger(__name__)
 
 
 class AdvancedChatAppGenerator(MessageBasedAppGenerator):
-    def generate(self, app_model: App,
-                 workflow: Workflow,
-                 user: Union[Account, EndUser],
-                 args: dict,
-                 invoke_from: InvokeFrom,
-                 stream: bool = True) \
-            -> Union[dict, Generator[dict, None, None]]:
+    def generate(
+        self, app_model: App,
+        workflow: Workflow,
+        user: Union[Account, EndUser],
+        args: dict,
+        invoke_from: InvokeFrom,
+        stream: bool = True,
+    ) -> Union[dict, Generator[dict, None, None]]:
         """
         Generate App response.
 
@@ -57,7 +59,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
         inputs = args['inputs']
 
         extras = {
-            "auto_generate_conversation_name": args['auto_generate_name'] if 'auto_generate_name' in args else False
+            "auto_generate_conversation_name": args.get('auto_generate_name', False)
         }
 
         # get conversation
@@ -84,6 +86,13 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             workflow=workflow
         )
 
+        # get tracing instance
+        trace_manager = TraceQueueManager(app_id=app_model.id)
+
+        if invoke_from == InvokeFrom.DEBUGGER:
+            # always enable retriever resource in debugger mode
+            app_config.additional_features.show_retrieve_source = True
+
         # init application generate entity
         application_generate_entity = AdvancedChatAppGenerateEntity(
             task_id=str(uuid.uuid4()),
@@ -95,7 +104,8 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             user_id=user.id,
             stream=stream,
             invoke_from=invoke_from,
-            extras=extras
+            extras=extras,
+            trace_manager=trace_manager
         )
 
         return self._generate(

api/core/app/apps/advanced_chat/app_runner.py

@@ -70,7 +70,8 @@ class AdvancedChatAppRunner(AppRunner):
                 app_record=app_record,
                 app_generate_entity=application_generate_entity,
                 inputs=inputs,
-                query=query
+                query=query,
+                message_id=message.id
         ):
             return
 
@@ -156,11 +157,14 @@ class AdvancedChatAppRunner(AppRunner):
         # return workflow
         return workflow
 
-    def handle_input_moderation(self, queue_manager: AppQueueManager,
-                                app_record: App,
-                                app_generate_entity: AdvancedChatAppGenerateEntity,
-                                inputs: dict,
-                                query: str) -> bool:
+    def handle_input_moderation(
+            self, queue_manager: AppQueueManager,
+            app_record: App,
+            app_generate_entity: AdvancedChatAppGenerateEntity,
+            inputs: dict,
+            query: str,
+            message_id: str
+    ) -> bool:
         """
         Handle input moderation
         :param queue_manager: application queue manager
@@ -168,6 +172,7 @@ class AdvancedChatAppRunner(AppRunner):
         :param app_generate_entity: application generate entity
         :param inputs: inputs
         :param query: query
+        :param message_id: message id
         :return:
         """
         try:
@@ -178,6 +183,7 @@ class AdvancedChatAppRunner(AppRunner):
                 app_generate_entity=app_generate_entity,
                 inputs=inputs,
                 query=query,
+                message_id=message_id,
             )
         except ModerationException as e:
             self._stream_output(

api/core/app/apps/advanced_chat/generate_task_pipeline.py

@@ -42,6 +42,7 @@ from core.app.task_pipeline.workflow_cycle_manage import WorkflowCycleManage
 from core.file.file_obj import FileVar
 from core.model_runtime.entities.llm_entities import LLMUsage
 from core.model_runtime.utils.encoders import jsonable_encoder
+from core.ops.ops_trace_manager import TraceQueueManager
 from core.workflow.entities.node_entities import NodeType, SystemVariable
 from core.workflow.nodes.answer.answer_node import AnswerNode
 from core.workflow.nodes.answer.entities import TextGenerateRouteChunk, VarGenerateRouteChunk
@@ -69,13 +70,15 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
     _workflow_system_variables: dict[SystemVariable, Any]
     _iteration_nested_relations: dict[str, list[str]]
 
-    def __init__(self, application_generate_entity: AdvancedChatAppGenerateEntity,
-                 workflow: Workflow,
-                 queue_manager: AppQueueManager,
-                 conversation: Conversation,
-                 message: Message,
-                 user: Union[Account, EndUser],
-                 stream: bool) -> None:
+    def __init__(
+        self, application_generate_entity: AdvancedChatAppGenerateEntity,
+        workflow: Workflow,
+        queue_manager: AppQueueManager,
+        conversation: Conversation,
+        message: Message,
+        user: Union[Account, EndUser],
+        stream: bool
+    ) -> None:
         """
         Initialize AdvancedChatAppGenerateTaskPipeline.
         :param application_generate_entity: application generate entity
@@ -126,14 +129,16 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
             self._application_generate_entity.query
         )
 
-        generator = self._process_stream_response()
+        generator = self._process_stream_response(
+            trace_manager=self._application_generate_entity.trace_manager
+        )
         if self._stream:
             return self._to_stream_response(generator)
         else:
             return self._to_blocking_response(generator)
 
     def _to_blocking_response(self, generator: Generator[StreamResponse, None, None]) \
-            -> ChatbotAppBlockingResponse:
+        -> ChatbotAppBlockingResponse:
         """
         Process blocking response.
         :return:
@@ -164,7 +169,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
         raise Exception('Queue listening stopped unexpectedly.')
 
     def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) \
-            -> Generator[ChatbotAppStreamResponse, None, None]:
+        -> Generator[ChatbotAppStreamResponse, None, None]:
         """
         To stream response.
         :return:
@@ -177,7 +182,9 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                 stream_response=stream_response
             )
 
-    def _process_stream_response(self) -> Generator[StreamResponse, None, None]:
+    def _process_stream_response(
+        self, trace_manager: Optional[TraceQueueManager] = None
+    ) -> Generator[StreamResponse, None, None]:
         """
         Process stream response.
         :return:
@@ -249,7 +256,9 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                 yield self._handle_iteration_to_stream_response(self._application_generate_entity.task_id, event)
                 self._handle_iteration_operation(event)
             elif isinstance(event, QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent):
-                workflow_run = self._handle_workflow_finished(event)
+                workflow_run = self._handle_workflow_finished(
+                    event, conversation_id=self._conversation.id, trace_manager=trace_manager
+                )
                 if workflow_run:
                     yield self._workflow_finish_to_stream_response(
                         task_id=self._application_generate_entity.task_id,
@@ -292,7 +301,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                     continue
 
                 if not self._is_stream_out_support(
-                        event=event
+                    event=event
                 ):
                     continue
 
@@ -361,7 +370,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
             id=self._message.id,
             **extras
         )
-    
+
     def _get_stream_generate_routes(self) -> dict[str, ChatflowStreamGenerateRoute]:
         """
         Get stream generate routes.
@@ -391,9 +400,9 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                 )
 
         return stream_generate_routes
-    
+
     def _get_answer_start_at_node_ids(self, graph: dict, target_node_id: str) \
-            -> list[str]:
+        -> list[str]:
         """
         Get answer start at node id.
         :param graph: graph
@@ -414,14 +423,14 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
             target_node = next((node for node in nodes if node.get('id') == target_node_id), None)
             if not target_node:
                 return []
-            
+
             node_iteration_id = target_node.get('data', {}).get('iteration_id')
             # get iteration start node id
             for node in nodes:
                 if node.get('id') == node_iteration_id:
                     if node.get('data', {}).get('start_node_id') == target_node_id:
                         return [target_node_id]
-                    
+
             return []
 
         start_node_ids = []
@@ -457,7 +466,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                     start_node_ids.extend(sub_start_node_ids)
 
         return start_node_ids
-    
+
     def _get_iteration_nested_relations(self, graph: dict) -> dict[str, list[str]]:
         """
         Get iteration nested relations.
@@ -466,18 +475,18 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
         """
         nodes = graph.get('nodes')
 
-        iteration_ids = [node.get('id') for node in nodes 
+        iteration_ids = [node.get('id') for node in nodes
                          if node.get('data', {}).get('type') in [
                              NodeType.ITERATION.value,
                              NodeType.LOOP.value,
-                        ]]
+                         ]]
 
         return {
             iteration_id: [
                 node.get('id') for node in nodes if node.get('data', {}).get('iteration_id') == iteration_id
             ] for iteration_id in iteration_ids
         }
-    
+
     def _generate_stream_outputs_when_node_started(self) -> Generator:
         """
         Generate stream outputs.
@@ -485,8 +494,8 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
         """
         if self._task_state.current_stream_generate_state:
             route_chunks = self._task_state.current_stream_generate_state.generate_route[
-                self._task_state.current_stream_generate_state.current_route_position:
-            ]
+                           self._task_state.current_stream_generate_state.current_route_position:
+                           ]
 
             for route_chunk in route_chunks:
                 if route_chunk.type == 'text':
@@ -506,7 +515,8 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
 
             # all route chunks are generated
             if self._task_state.current_stream_generate_state.current_route_position == len(
-                    self._task_state.current_stream_generate_state.generate_route):
+                self._task_state.current_stream_generate_state.generate_route
+            ):
                 self._task_state.current_stream_generate_state = None
 
     def _generate_stream_outputs_when_node_finished(self) -> Optional[Generator]:
@@ -519,7 +529,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
 
         route_chunks = self._task_state.current_stream_generate_state.generate_route[
                        self._task_state.current_stream_generate_state.current_route_position:]
-        
+
         for route_chunk in route_chunks:
             if route_chunk.type == 'text':
                 route_chunk = cast(TextGenerateRouteChunk, route_chunk)
@@ -551,7 +561,8 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                         value = iteration_state.current_index
                     elif value_selector[1] == 'item':
                         value = iterator_selector[iteration_state.current_index] if iteration_state.current_index < len(
-                            iterator_selector) else None
+                            iterator_selector
+                        ) else None
                 else:
                     # check chunk node id is before current node id or equal to current node id
                     if route_chunk_node_id not in self._task_state.ran_node_execution_infos:
@@ -562,14 +573,15 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                     # get route chunk node execution info
                     route_chunk_node_execution_info = self._task_state.ran_node_execution_infos[route_chunk_node_id]
                     if (route_chunk_node_execution_info.node_type == NodeType.LLM
-                            and latest_node_execution_info.node_type == NodeType.LLM):
+                        and latest_node_execution_info.node_type == NodeType.LLM):
                         # only LLM support chunk stream output
                         self._task_state.current_stream_generate_state.current_route_position += 1
                         continue
 
                     # get route chunk node execution
                     route_chunk_node_execution = db.session.query(WorkflowNodeExecution).filter(
-                        WorkflowNodeExecution.id == route_chunk_node_execution_info.workflow_node_execution_id).first()
+                        WorkflowNodeExecution.id == route_chunk_node_execution_info.workflow_node_execution_id
+                    ).first()
 
                     outputs = route_chunk_node_execution.outputs_dict
 
@@ -631,7 +643,8 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
 
         # all route chunks are generated
         if self._task_state.current_stream_generate_state.current_route_position == len(
-                self._task_state.current_stream_generate_state.generate_route):
+            self._task_state.current_stream_generate_state.generate_route
+        ):
             self._task_state.current_stream_generate_state = None
 
     def _is_stream_out_support(self, event: QueueTextChunkEvent) -> bool:

api/core/app/apps/agent_chat/app_generator.py

@@ -19,6 +19,7 @@ from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueMa
 from core.app.entities.app_invoke_entities import AgentChatAppGenerateEntity, InvokeFrom
 from core.file.message_file_parser import MessageFileParser
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
+from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
 from models.account import Account
 from models.model import App, EndUser
@@ -56,7 +57,7 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
         inputs = args['inputs']
 
         extras = {
-            "auto_generate_conversation_name": args['auto_generate_name'] if 'auto_generate_name' in args else True
+            "auto_generate_conversation_name": args.get('auto_generate_name', True)
         }
 
         # get conversation
@@ -82,6 +83,11 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
                 config=args.get('model_config')
             )
 
+            # always enable retriever resource in debugger mode
+            override_model_config_dict["retriever_resource"] = {
+                "enabled": True
+            }
+
         # parse files
         files = args['files'] if args.get('files') else []
         message_file_parser = MessageFileParser(tenant_id=app_model.tenant_id, app_id=app_model.id)
@@ -103,6 +109,9 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
             override_config_dict=override_model_config_dict
         )
 
+        # get tracing instance
+        trace_manager = TraceQueueManager(app_model.id)
+
         # init application generate entity
         application_generate_entity = AgentChatAppGenerateEntity(
             task_id=str(uuid.uuid4()),
@@ -116,7 +125,8 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
             stream=stream,
             invoke_from=invoke_from,
             extras=extras,
-            call_depth=0
+            call_depth=0,
+            trace_manager=trace_manager
         )
 
         # init generate records
@@ -153,7 +163,7 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
             conversation=conversation,
             message=message,
             user=user,
-            stream=stream
+            stream=stream,
         )
 
         return AgentChatAppGenerateResponseConverter.convert(
@@ -161,11 +171,13 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
             invoke_from=invoke_from
         )
 
-    def _generate_worker(self, flask_app: Flask,
-                         application_generate_entity: AgentChatAppGenerateEntity,
-                         queue_manager: AppQueueManager,
-                         conversation_id: str,
-                         message_id: str) -> None:
+    def _generate_worker(
+        self, flask_app: Flask,
+        application_generate_entity: AgentChatAppGenerateEntity,
+        queue_manager: AppQueueManager,
+        conversation_id: str,
+        message_id: str,
+    ) -> None:
         """
         Generate worker in a new thread.
         :param flask_app: Flask app
@@ -187,7 +199,7 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
                     application_generate_entity=application_generate_entity,
                     queue_manager=queue_manager,
                     conversation=conversation,
-                    message=message
+                    message=message,
                 )
             except GenerateTaskStoppedException:
                 pass

api/core/app/apps/agent_chat/app_runner.py

@@ -28,10 +28,13 @@ class AgentChatAppRunner(AppRunner):
     """
     Agent Application Runner
     """
-    def run(self, application_generate_entity: AgentChatAppGenerateEntity,
-            queue_manager: AppQueueManager,
-            conversation: Conversation,
-            message: Message) -> None:
+
+    def run(
+        self, application_generate_entity: AgentChatAppGenerateEntity,
+        queue_manager: AppQueueManager,
+        conversation: Conversation,
+        message: Message,
+    ) -> None:
         """
         Run assistant application
         :param application_generate_entity: application generate entity
@@ -100,6 +103,7 @@ class AgentChatAppRunner(AppRunner):
                 app_generate_entity=application_generate_entity,
                 inputs=inputs,
                 query=query,
+                message_id=message.id
             )
         except ModerationException as e:
             self.direct_output(
@@ -199,7 +203,7 @@ class AgentChatAppRunner(AppRunner):
         llm_model = cast(LargeLanguageModel, model_instance.model_type_instance)
         model_schema = llm_model.get_model_schema(model_instance.model, model_instance.credentials)
 
-        if set([ModelFeature.MULTI_TOOL_CALL, ModelFeature.TOOL_CALL]).intersection(model_schema.features or []):
+        if {ModelFeature.MULTI_TOOL_CALL, ModelFeature.TOOL_CALL}.intersection(model_schema.features or []):
             agent_entity.strategy = AgentEntity.Strategy.FUNCTION_CALLING
 
         conversation = db.session.query(Conversation).filter(Conversation.id == conversation.id).first()
@@ -219,7 +223,7 @@ class AgentChatAppRunner(AppRunner):
             runner_cls = FunctionCallAgentRunner
         else:
             raise ValueError(f"Invalid agent strategy: {agent_entity.strategy}")
-        
+
         runner = runner_cls(
             tenant_id=app_config.tenant_id,
             application_generate_entity=application_generate_entity,

api/core/app/apps/base_app_runner.py

@@ -338,11 +338,14 @@ class AppRunner:
             ), PublishFrom.APPLICATION_MANAGER
         )
 
-    def moderation_for_inputs(self, app_id: str,
-                              tenant_id: str,
-                              app_generate_entity: AppGenerateEntity,
-                              inputs: dict,
-                              query: str) -> tuple[bool, dict, str]:
+    def moderation_for_inputs(
+            self, app_id: str,
+            tenant_id: str,
+            app_generate_entity: AppGenerateEntity,
+            inputs: dict,
+            query: str,
+            message_id: str,
+    ) -> tuple[bool, dict, str]:
         """
         Process sensitive_word_avoidance.
         :param app_id: app id
@@ -350,6 +353,7 @@ class AppRunner:
         :param app_generate_entity: app generate entity
         :param inputs: inputs
         :param query: query
+        :param message_id: message id
         :return:
         """
         moderation_feature = InputModeration()
@@ -358,7 +362,9 @@ class AppRunner:
             tenant_id=tenant_id,
             app_config=app_generate_entity.app_config,
             inputs=inputs,
-            query=query if query else ''
+            query=query if query else '',
+            message_id=message_id,
+            trace_manager=app_generate_entity.trace_manager
         )
     
     def check_hosting_moderation(self, application_generate_entity: EasyUIBasedAppGenerateEntity,

api/core/app/apps/chat/app_config_manager.py

@@ -50,6 +50,9 @@ class ChatAppConfigManager(BaseAppConfigManager):
             app_model_config_dict = app_model_config.to_dict()
             config_dict = app_model_config_dict.copy()
         else:
+            if not override_config_dict:
+                raise Exception('override_config_dict is required when config_from is ARGS')
+
             config_dict = override_config_dict
 
         app_mode = AppMode.value_of(app_model.mode)

api/core/app/apps/chat/app_generator.py

@@ -19,6 +19,7 @@ from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueMa
 from core.app.entities.app_invoke_entities import ChatAppGenerateEntity, InvokeFrom
 from core.file.message_file_parser import MessageFileParser
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
+from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
 from models.account import Account
 from models.model import App, EndUser
@@ -27,12 +28,13 @@ logger = logging.getLogger(__name__)
 
 
 class ChatAppGenerator(MessageBasedAppGenerator):
-    def generate(self, app_model: App,
-                 user: Union[Account, EndUser],
-                 args: Any,
-                 invoke_from: InvokeFrom,
-                 stream: bool = True) \
-            -> Union[dict, Generator[dict, None, None]]:
+    def generate(
+        self, app_model: App,
+        user: Union[Account, EndUser],
+        args: Any,
+        invoke_from: InvokeFrom,
+        stream: bool = True,
+    ) -> Union[dict, Generator[dict, None, None]]:
         """
         Generate App response.
 
@@ -53,7 +55,7 @@ class ChatAppGenerator(MessageBasedAppGenerator):
         inputs = args['inputs']
 
         extras = {
-            "auto_generate_conversation_name": args['auto_generate_name'] if 'auto_generate_name' in args else True
+            "auto_generate_conversation_name": args.get('auto_generate_name', True)
         }
 
         # get conversation
@@ -79,6 +81,11 @@ class ChatAppGenerator(MessageBasedAppGenerator):
                 config=args.get('model_config')
             )
 
+            # always enable retriever resource in debugger mode
+            override_model_config_dict["retriever_resource"] = {
+                "enabled": True
+            }
+
         # parse files
         files = args['files'] if args.get('files') else []
         message_file_parser = MessageFileParser(tenant_id=app_model.tenant_id, app_id=app_model.id)
@@ -100,6 +107,9 @@ class ChatAppGenerator(MessageBasedAppGenerator):
             override_config_dict=override_model_config_dict
         )
 
+        # get tracing instance
+        trace_manager = TraceQueueManager(app_model.id)
+
         # init application generate entity
         application_generate_entity = ChatAppGenerateEntity(
             task_id=str(uuid.uuid4()),
@@ -112,7 +122,8 @@ class ChatAppGenerator(MessageBasedAppGenerator):
             user_id=user.id,
             stream=stream,
             invoke_from=invoke_from,
-            extras=extras
+            extras=extras,
+            trace_manager=trace_manager
         )
 
         # init generate records
@@ -149,7 +160,7 @@ class ChatAppGenerator(MessageBasedAppGenerator):
             conversation=conversation,
             message=message,
             user=user,
-            stream=stream
+            stream=stream,
         )
 
         return ChatAppGenerateResponseConverter.convert(

api/core/app/apps/chat/app_runner.py

@@ -96,6 +96,7 @@ class ChatAppRunner(AppRunner):
                 app_generate_entity=application_generate_entity,
                 inputs=inputs,
                 query=query,
+                message_id=message.id
             )
         except ModerationException as e:
             self.direct_output(
@@ -154,7 +155,7 @@ class ChatAppRunner(AppRunner):
                 application_generate_entity.invoke_from
             )
 
-            dataset_retrieval = DatasetRetrieval()
+            dataset_retrieval = DatasetRetrieval(application_generate_entity)
             context = dataset_retrieval.retrieve(
                 app_id=app_record.id,
                 user_id=application_generate_entity.user_id,
@@ -165,7 +166,8 @@ class ChatAppRunner(AppRunner):
                 invoke_from=application_generate_entity.invoke_from,
                 show_retrieve_source=app_config.additional_features.show_retrieve_source,
                 hit_callback=hit_callback,
-                memory=memory
+                memory=memory,
+                message_id=message.id,
             )
 
         # reorganize all inputs and template to prompt messages

api/core/app/apps/completion/app_generator.py

@@ -19,6 +19,7 @@ from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueMa
 from core.app.entities.app_invoke_entities import CompletionAppGenerateEntity, InvokeFrom
 from core.file.message_file_parser import MessageFileParser
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
+from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
 from models.account import Account
 from models.model import App, EndUser, Message
@@ -94,6 +95,9 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
             override_config_dict=override_model_config_dict
         )
 
+        # get tracing instance
+        trace_manager = TraceQueueManager(app_model.id)
+
         # init application generate entity
         application_generate_entity = CompletionAppGenerateEntity(
             task_id=str(uuid.uuid4()),
@@ -105,7 +109,8 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
             user_id=user.id,
             stream=stream,
             invoke_from=invoke_from,
-            extras=extras
+            extras=extras,
+            trace_manager=trace_manager
         )
 
         # init generate records
@@ -141,7 +146,7 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
             conversation=conversation,
             message=message,
             user=user,
-            stream=stream
+            stream=stream,
         )
 
         return CompletionAppGenerateResponseConverter.convert(
@@ -158,7 +163,6 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
         :param flask_app: Flask app
         :param application_generate_entity: application generate entity
         :param queue_manager: queue manager
-        :param conversation_id: conversation ID
         :param message_id: message ID
         :return:
         """
@@ -300,7 +304,7 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
             conversation=conversation,
             message=message,
             user=user,
-            stream=stream
+            stream=stream,
         )
 
         return CompletionAppGenerateResponseConverter.convert(

api/core/app/apps/completion/app_runner.py

@@ -77,6 +77,7 @@ class CompletionAppRunner(AppRunner):
                 app_generate_entity=application_generate_entity,
                 inputs=inputs,
                 query=query,
+                message_id=message.id
             )
         except ModerationException as e:
             self.direct_output(
@@ -114,7 +115,7 @@ class CompletionAppRunner(AppRunner):
             if dataset_config and dataset_config.retrieve_config.query_variable:
                 query = inputs.get(dataset_config.retrieve_config.query_variable, "")
 
-            dataset_retrieval = DatasetRetrieval()
+            dataset_retrieval = DatasetRetrieval(application_generate_entity)
             context = dataset_retrieval.retrieve(
                 app_id=app_record.id,
                 user_id=application_generate_entity.user_id,
@@ -124,7 +125,8 @@ class CompletionAppRunner(AppRunner):
                 query=query,
                 invoke_from=application_generate_entity.invoke_from,
                 show_retrieve_source=app_config.additional_features.show_retrieve_source,
-                hit_callback=hit_callback
+                hit_callback=hit_callback,
+                message_id=message.id
             )
 
         # reorganize all inputs and template to prompt messages

api/core/app/apps/message_based_app_generator.py

@@ -35,22 +35,23 @@ logger = logging.getLogger(__name__)
 
 class MessageBasedAppGenerator(BaseAppGenerator):
 
-    def _handle_response(self, application_generate_entity: Union[
-        ChatAppGenerateEntity,
-        CompletionAppGenerateEntity,
-        AgentChatAppGenerateEntity,
-        AdvancedChatAppGenerateEntity
-    ],
-                         queue_manager: AppQueueManager,
-                         conversation: Conversation,
-                         message: Message,
-                         user: Union[Account, EndUser],
-                         stream: bool = False) \
-            -> Union[
-                ChatbotAppBlockingResponse,
-                CompletionAppBlockingResponse,
-                Generator[Union[ChatbotAppStreamResponse, CompletionAppStreamResponse], None, None]
-            ]:
+    def _handle_response(
+        self, application_generate_entity: Union[
+            ChatAppGenerateEntity,
+            CompletionAppGenerateEntity,
+            AgentChatAppGenerateEntity,
+            AdvancedChatAppGenerateEntity
+        ],
+        queue_manager: AppQueueManager,
+        conversation: Conversation,
+        message: Message,
+        user: Union[Account, EndUser],
+        stream: bool = False,
+    ) -> Union[
+        ChatbotAppBlockingResponse,
+        CompletionAppBlockingResponse,
+        Generator[Union[ChatbotAppStreamResponse, CompletionAppStreamResponse], None, None]
+    ]:
         """
         Handle response.
         :param application_generate_entity: application generate entity

api/core/app/apps/workflow/app_generator.py

@@ -20,6 +20,7 @@ from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerat
 from core.app.entities.task_entities import WorkflowAppBlockingResponse, WorkflowAppStreamResponse
 from core.file.message_file_parser import MessageFileParser
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
+from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
 from models.account import Account
 from models.model import App, EndUser
@@ -29,14 +30,15 @@ logger = logging.getLogger(__name__)
 
 
 class WorkflowAppGenerator(BaseAppGenerator):
-    def generate(self, app_model: App,
-                 workflow: Workflow,
-                 user: Union[Account, EndUser],
-                 args: dict,
-                 invoke_from: InvokeFrom,
-                 stream: bool = True,
-                 call_depth: int = 0) \
-            -> Union[dict, Generator[dict, None, None]]:
+    def generate(
+        self, app_model: App,
+        workflow: Workflow,
+        user: Union[Account, EndUser],
+        args: dict,
+        invoke_from: InvokeFrom,
+        stream: bool = True,
+        call_depth: int = 0,
+    ) -> Union[dict, Generator[dict, None, None]]:
         """
         Generate App response.
 
@@ -46,6 +48,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
         :param args: request args
         :param invoke_from: invoke from source
         :param stream: is stream
+        :param call_depth: call depth
         """
         inputs = args['inputs']
 
@@ -68,6 +71,9 @@ class WorkflowAppGenerator(BaseAppGenerator):
             workflow=workflow
         )
 
+        # get tracing instance
+        trace_manager = TraceQueueManager(app_model.id)
+
         # init application generate entity
         application_generate_entity = WorkflowAppGenerateEntity(
             task_id=str(uuid.uuid4()),
@@ -77,7 +83,8 @@ class WorkflowAppGenerator(BaseAppGenerator):
             user_id=user.id,
             stream=stream,
             invoke_from=invoke_from,
-            call_depth=call_depth
+            call_depth=call_depth,
+            trace_manager=trace_manager
         )
 
         return self._generate(
@@ -87,17 +94,18 @@ class WorkflowAppGenerator(BaseAppGenerator):
             application_generate_entity=application_generate_entity,
             invoke_from=invoke_from,
             stream=stream,
-            call_depth=call_depth
+            call_depth=call_depth,
         )
 
-    def _generate(self, app_model: App,
-                 workflow: Workflow,
-                 user: Union[Account, EndUser],
-                 application_generate_entity: WorkflowAppGenerateEntity,
-                 invoke_from: InvokeFrom,
-                 stream: bool = True,
-                 call_depth: int = 0) \
-            -> Union[dict, Generator[dict, None, None]]:
+    def _generate(
+        self, app_model: App,
+        workflow: Workflow,
+        user: Union[Account, EndUser],
+        application_generate_entity: WorkflowAppGenerateEntity,
+        invoke_from: InvokeFrom,
+        stream: bool = True,
+        call_depth: int = 0
+    ) -> Union[dict, Generator[dict, None, None]]:
         """
         Generate App response.
 
@@ -131,7 +139,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
             workflow=workflow,
             queue_manager=queue_manager,
             user=user,
-            stream=stream
+            stream=stream,
         )
 
         return WorkflowAppGenerateResponseConverter.convert(

api/core/app/apps/workflow/generate_task_pipeline.py

@@ -1,6 +1,6 @@
 import logging
 from collections.abc import Generator
-from typing import Any, Union
+from typing import Any, Optional, Union
 
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import (
@@ -36,6 +36,7 @@ from core.app.entities.task_entities import (
 )
 from core.app.task_pipeline.based_generate_task_pipeline import BasedGenerateTaskPipeline
 from core.app.task_pipeline.workflow_cycle_manage import WorkflowCycleManage
+from core.ops.ops_trace_manager import TraceQueueManager
 from core.workflow.entities.node_entities import NodeType, SystemVariable
 from core.workflow.nodes.end.end_node import EndNode
 from extensions.ext_database import db
@@ -104,7 +105,9 @@ class WorkflowAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCycleMa
         db.session.refresh(self._user)
         db.session.close()
 
-        generator = self._process_stream_response()
+        generator = self._process_stream_response(
+            trace_manager=self._application_generate_entity.trace_manager
+        )
         if self._stream:
             return self._to_stream_response(generator)
         else:
@@ -158,7 +161,10 @@ class WorkflowAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCycleMa
                 stream_response=stream_response
             )
 
-    def _process_stream_response(self) -> Generator[StreamResponse, None, None]:
+    def _process_stream_response(
+        self,
+        trace_manager: Optional[TraceQueueManager] = None
+    ) -> Generator[StreamResponse, None, None]:
         """
         Process stream response.
         :return:
@@ -215,7 +221,9 @@ class WorkflowAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCycleMa
                 yield self._handle_iteration_to_stream_response(self._application_generate_entity.task_id, event)
                 self._handle_iteration_operation(event)
             elif isinstance(event, QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent):
-                workflow_run = self._handle_workflow_finished(event)
+                workflow_run = self._handle_workflow_finished(
+                    event, trace_manager=trace_manager
+                )
 
                 # save workflow app log
                 self._save_workflow_app_log(workflow_run)

api/core/app/entities/app_invoke_entities.py

@@ -7,6 +7,7 @@ from core.app.app_config.entities import AppConfig, EasyUIBasedAppConfig, Workfl
 from core.entities.provider_configuration import ProviderModelBundle
 from core.file.file_obj import FileVar
 from core.model_runtime.entities.model_entities import AIModelEntity
+from core.ops.ops_trace_manager import TraceQueueManager
 
 
 class InvokeFrom(Enum):
@@ -89,6 +90,12 @@ class AppGenerateEntity(BaseModel):
     # extra parameters, like: auto_generate_conversation_name
     extras: dict[str, Any] = {}
 
+    # tracing instance
+    trace_manager: Optional[TraceQueueManager] = None
+
+    class Config:
+        arbitrary_types_allowed = True
+
 
 class EasyUIBasedAppGenerateEntity(AppGenerateEntity):
     """

api/core/app/entities/queue_entities.py

@@ -77,8 +77,8 @@ class QueueIterationNextEvent(AppQueueEvent):
     node_run_index: int
     output: Optional[Any] = None # output for the current iteration
 
-    @classmethod
     @field_validator('output', mode='before')
+    @classmethod
     def set_output(cls, v):
         """
         Set output

api/core/app/entities/task_entities.py

@@ -367,7 +367,7 @@ class IterationNodeNextStreamResponse(StreamResponse):
 
 class IterationNodeCompletedStreamResponse(StreamResponse):
     """
-    NodeStartStreamResponse entity
+    NodeCompletedStreamResponse entity
     """
     class Data(BaseModel):
         """
@@ -385,6 +385,7 @@ class IterationNodeCompletedStreamResponse(StreamResponse):
         error: Optional[str] = None
         elapsed_time: float
         total_tokens: int
+        execution_metadata: Optional[dict] = None
         finished_at: int
         steps: int
 
@@ -545,4 +546,4 @@ class WorkflowIterationState(BaseModel):
         total_tokens: int = 0
         node_data: BaseNodeData
 
-    current_iterations: dict[str, Data] = None
+    current_iterations: dict[str, Data] = None