feat: 白名单中去除 maas 平台

.devcontainer/post_create_command.sh

@@ -3,7 +3,7 @@
 cd web && npm install
 
 echo 'alias start-api="cd /workspaces/dify/api && flask run --host 0.0.0.0 --port=5001 --debug"' >> ~/.bashrc
-echo 'alias start-worker="cd /workspaces/dify/api && celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion"' >> ~/.bashrc
+echo 'alias start-worker="cd /workspaces/dify/api && celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail"' >> ~/.bashrc
 echo 'alias start-web="cd /workspaces/dify/web && npm run dev"' >> ~/.bashrc
 echo 'alias start-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify up -d"' >> ~/.bashrc
 

.devcontainer/post_start_command.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-poetry install -C api
+cd api && pip install -r requirements.txt

.gitattributes

@@ -1,7 +0,0 @@
-# Ensure that .sh scripts use LF as line separator, even if they are checked out
-# to Windows(NTFS) file-system, by a user of Docker for Window. 
-# These .sh scripts will be run from the Container after `docker compose up -d`.
-# If they appear to be CRLF style, Dash from the Container will fail to execute
-# them.
-
-*.sh      text eol=lf

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -14,15 +14,13 @@ body:
           required: true
         - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
-        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
-          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
 
   - type: input
     attributes:
       label: Dify version
-      placeholder: 0.6.11
+      placeholder: 0.3.21
       description: See about section in Dify console
     validations:
       required: true
@@ -42,7 +40,7 @@ body:
   - type: textarea
     attributes:
       label: Steps to reproduce
-      description: We highly suggest including screenshots and a bug report log. Please use the right markdown syntax for code blocks.
+      description: We highly suggest including screenshots and a bug report log.
       placeholder: Having detailed steps helps us reproduce the bug.
     validations:
       required: true

.github/ISSUE_TEMPLATE/document_issue.yml

@@ -12,8 +12,6 @@ body:
           required: true
         - label: I confirm that I am using English to submit report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
-        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
-          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: textarea

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -12,25 +12,35 @@ body:
           required: true
         - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
-        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
-          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: textarea
     attributes:
-      label: 1. Is this request related to a challenge you're experiencing? Tell me about your story.
+      label: 1. Is this request related to a challenge you're experiencing?
       placeholder: Please describe the specific scenario or problem you're facing as clearly as possible. For instance "I was trying to use [feature] for [specific task], and [what happened]... It was frustrating because...."
     validations:
       required: true
   - type: textarea
     attributes:
-      label: 2. Additional context or comments
+      label: 2. Describe the feature you'd like to see
+      placeholder: Think about what you want to achieve and how this feature will help you. Sketches, flow diagrams, or any visual representation will be a major plus.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: 3. How will this feature improve your workflow or experience?
+      placeholder: Tell us how this change will benefit your work. This helps us prioritize based on user impact.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: 4. Additional context or comments
       placeholder: (Any other information, comments, documentations, links, or screenshots that would provide more clarity. This is the place to add anything else not covered above.)
     validations:
       required: false
   - type: checkboxes
     attributes:
-      label: 3. Can you help us with this feature?
+      label: 5. Can you help us with this feature?
       description: Let us know! This is not a commitment, but a starting point for collaboration.
       options:
         - label: I am interested in contributing to this feature.

.github/ISSUE_TEMPLATE/translation_issue.yml

@@ -12,8 +12,6 @@ body:
           required: true
         - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
-        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
-          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: input

.github/workflows/api-tests.yml

@@ -14,6 +14,74 @@ concurrency:
 
 jobs:
   test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.10"
+          - "3.11"
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: |
+            ./api/requirements.txt
+            ./api/requirements-dev.txt
+
+      - name: Install dependencies
+        run: pip install -r ./api/requirements.txt -r ./api/requirements-dev.txt
+
+      - name: Run Unit tests
+        run: dev/pytest/pytest_unit_tests.sh
+
+      - name: Run ModelRuntime
+        run: dev/pytest/pytest_model_runtime.sh
+
+      - name: Run Tool
+        run: dev/pytest/pytest_tools.sh
+
+      - name: Set up Sandbox
+        uses: hoverkraft-tech/compose-action@v2.0.0
+        with:
+          compose-file: |
+            docker/docker-compose.middleware.yaml
+          services: |
+            sandbox
+            ssrf_proxy
+
+      - name: Run Workflow
+        run: dev/pytest/pytest_workflow.sh
+
+      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma)
+        uses: hoverkraft-tech/compose-action@v2.0.0
+        with:
+          compose-file: |
+            docker/docker-compose.middleware.yaml
+            docker/docker-compose.qdrant.yaml
+            docker/docker-compose.milvus.yaml
+            docker/docker-compose.pgvecto-rs.yaml
+            docker/docker-compose.pgvector.yaml
+            docker/docker-compose.chroma.yaml
+          services: |
+            weaviate
+            qdrant
+            etcd
+            minio
+            milvus-standalone
+            pgvecto-rs
+            pgvector
+            chroma
+
+      - name: Test Vector Stores
+        run: dev/pytest/pytest_vdb.sh
+
+  test-in-poetry:
     name: API Tests
     runs-on: ubuntu-latest
     strategy:
@@ -40,7 +108,7 @@ jobs:
 
       - name: Poetry check
         run: |
-          poetry check -C api --lock
+          poetry check -C api
           poetry show -C api
 
       - name: Install dependencies
@@ -55,14 +123,6 @@ jobs:
       - name: Run Tool
         run: poetry run -C api bash dev/pytest/pytest_tools.sh
 
-      - name: Set up dotenvs
-        run: |
-          cp docker/.env.example docker/.env
-          cp docker/middleware.env.example docker/middleware.env
-
-      - name: Expose Service Ports
-        run: sh .github/workflows/expose_service_ports.sh
-
       - name: Set up Sandbox
         uses: hoverkraft-tech/compose-action@v2.0.0
         with:
@@ -79,7 +139,12 @@ jobs:
         uses: hoverkraft-tech/compose-action@v2.0.0
         with:
           compose-file: |
-            docker/docker-compose.yaml
+            docker/docker-compose.middleware.yaml
+            docker/docker-compose.qdrant.yaml
+            docker/docker-compose.milvus.yaml
+            docker/docker-compose.pgvecto-rs.yaml
+            docker/docker-compose.pgvector.yaml
+            docker/docker-compose.chroma.yaml
           services: |
             weaviate
             qdrant
@@ -89,5 +154,6 @@ jobs:
             pgvecto-rs
             pgvector
             chroma
+
       - name: Test Vector Stores
         run: poetry run -C api bash dev/pytest/pytest_vdb.sh

.github/workflows/build-push.yml

@@ -8,10 +8,6 @@ on:
   release:
     types: [published]
 
-concurrency:
-  group: build-push-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
 env:
   DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
   DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -19,35 +15,19 @@ env:
   DIFY_API_IMAGE_NAME: ${{ vars.DIFY_API_IMAGE_NAME || 'langgenius/dify-api' }}
 
 jobs:
-  build:
-    runs-on: ${{ matrix.platform == 'linux/arm64' && 'arm64_runner' || 'ubuntu-latest' }}
+  build-and-push:
+    runs-on: ubuntu-latest
     if: github.repository == 'langgenius/dify'
     strategy:
       matrix:
         include:
-          - service_name: "build-api-amd64"
-            image_name_env: "DIFY_API_IMAGE_NAME"
-            context: "api"
-            platform: linux/amd64
-          - service_name: "build-api-arm64"
-            image_name_env: "DIFY_API_IMAGE_NAME"
-            context: "api"
-            platform: linux/arm64
-          - service_name: "build-web-amd64"
+          - service_name: "web"
             image_name_env: "DIFY_WEB_IMAGE_NAME"
             context: "web"
-            platform: linux/amd64
-          - service_name: "build-web-arm64"
-            image_name_env: "DIFY_WEB_IMAGE_NAME"
-            context: "web"
-            platform: linux/arm64
-
+          - service_name: "api"
+            image_name_env: "DIFY_API_IMAGE_NAME"
+            context: "api"
     steps:
-      - name: Prepare
-        run: |
-          platform=${{ matrix.platform }}
-          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
@@ -60,66 +40,7 @@ jobs:
           username: ${{ env.DOCKERHUB_USER }}
           password: ${{ env.DOCKERHUB_TOKEN }}
 
-      - name: Extract metadata for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env[matrix.image_name_env] }}
-
-      - name: Build Docker image
-        id: build
-        uses: docker/build-push-action@v6
-        with:
-          context: "{{defaultContext}}:${{ matrix.context }}"
-          platforms: ${{ matrix.platform }}
-          build-args: COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
-          labels: ${{ steps.meta.outputs.labels }}
-          outputs: type=image,name=${{ env[matrix.image_name_env] }},push-by-digest=true,name-canonical=true,push=true
-          cache-from: type=gha,scope=${{ matrix.service_name }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.service_name }}
-
-      - name: Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-
-      - name: Upload digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: digests-${{ matrix.context }}-${{ env.PLATFORM_PAIR }}
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
-
-  create-manifest:
-    needs: build
-    runs-on: ubuntu-latest
-    if: github.repository == 'langgenius/dify'
-    strategy:
-      matrix:
-        include:
-          - service_name: "merge-api-images"
-            image_name_env: "DIFY_API_IMAGE_NAME"
-            context: "api"
-          - service_name: "merge-web-images"
-            image_name_env: "DIFY_WEB_IMAGE_NAME"
-            context: "web"
-    steps:
-      - name: Download digests
-        uses: actions/download-artifact@v4
-        with:
-          path: /tmp/digests
-          pattern: digests-${{ matrix.context }}-*
-          merge-multiple: true
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ env.DOCKERHUB_USER }}
-          password: ${{ env.DOCKERHUB_TOKEN }}
-
-      - name: Extract metadata for Docker
+      - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@v5
         with:
@@ -130,12 +51,14 @@ jobs:
             type=sha,enable=true,priority=100,prefix=,suffix=,format=long
             type=raw,value=${{ github.ref_name }},enable=${{ startsWith(github.ref, 'refs/tags/') }}
 
-      - name: Create manifest list and push
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ env[matrix.image_name_env] }}@sha256:%s ' *)
-
-      - name: Inspect image
-        run: |
-          docker buildx imagetools inspect ${{ env[matrix.image_name_env] }}:${{ steps.meta.outputs.version }}
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: "{{defaultContext}}:${{ matrix.context }}"
+          platforms: ${{ startsWith(github.ref, 'refs/tags/') && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
+          build-args: COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/db-migration-test.yml

@@ -38,11 +38,6 @@ jobs:
       - name: Install dependencies
         run: poetry install -C api
 
-      - name: Prepare middleware env
-        run: |
-          cd docker
-          cp middleware.env.example middleware.env
-
       - name: Set up Middlewares
         uses: hoverkraft-tech/compose-action@v2.0.0
         with:

.github/workflows/expose_service_ports.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-yq eval '.services.weaviate.ports += ["8080:8080"]' -i docker/docker-compose.yaml
-yq eval '.services.qdrant.ports += ["6333:6333"]' -i docker/docker-compose.yaml
-yq eval '.services.chroma.ports += ["8000:8000"]' -i docker/docker-compose.yaml
-yq eval '.services["milvus-standalone"].ports += ["19530:19530"]' -i docker/docker-compose.yaml
-yq eval '.services.pgvector.ports += ["5433:5432"]' -i docker/docker-compose.yaml
-yq eval '.services["pgvecto-rs"].ports += ["5431:5432"]' -i docker/docker-compose.yaml
-
-echo "Ports exposed for sandbox, weaviate, qdrant, chroma, milvus, pgvector, pgvecto-rs."

.github/workflows/style.yml

@@ -99,7 +99,7 @@ jobs:
             **.sh
             **.yaml
             **.yml
-            **Dockerfile
+            Dockerfile
             dev/**
 
       - name: Super-linter
@@ -113,8 +113,7 @@ jobs:
           IGNORE_GITIGNORED_FILES: true
           VALIDATE_BASH: true
           VALIDATE_BASH_EXEC: true
-          # FIXME: temporarily disabled until api-docker.yaml's run script is fixed for shellcheck
-          # VALIDATE_GITHUB_ACTIONS: true
+          VALIDATE_GITHUB_ACTIONS: true
           VALIDATE_DOCKERFILE_HADOLINT: true
           VALIDATE_XML: true
           VALIDATE_YAML: true

.gitignore

@@ -138,22 +138,10 @@ web/.vscode/settings.json
 !.idea/vcs.xml
 !.idea/icon.png
 .ideaDataSources/
-*.iml
-api/.idea
 
 api/.env
 api/storage/*
 
-docker-legacy/volumes/app/storage/*
-docker-legacy/volumes/db/data/*
-docker-legacy/volumes/redis/data/*
-docker-legacy/volumes/weaviate/*
-docker-legacy/volumes/qdrant/*
-docker-legacy/volumes/etcd/*
-docker-legacy/volumes/minio/*
-docker-legacy/volumes/milvus/*
-docker-legacy/volumes/chroma/*
-
 docker/volumes/app/storage/*
 docker/volumes/db/data/*
 docker/volumes/redis/data/*
@@ -164,9 +152,6 @@ docker/volumes/minio/*
 docker/volumes/milvus/*
 docker/volumes/chroma/*
 
-docker/nginx/conf.d/default.conf
-docker/middleware.env
-
 sdks/python-client/build
 sdks/python-client/dist
 sdks/python-client/dify_client.egg-info

README.md

@@ -174,7 +174,6 @@ The easiest way to start the Dify server is to run our [docker-compose.yml](dock
 
 ```bash
 cd docker
-cp .env.example .env
 docker compose up -d
 ```
 
@@ -184,7 +183,7 @@ After running, you can access the Dify dashboard in your browser at [http://loca
 
 ## Next steps
 
-If you need to customize the configuration, please refer to the comments in our [.env.example](docker/.env.example) file and update the corresponding values in your `.env` file. Additionally, you might need to make adjustments to the `docker-compose.yaml` file itself, such as changing image versions, port mappings, or volume mounts, based on your specific deployment environment and requirements. After making any changes, please re-run `docker-compose up -d`. You can find the full list of available environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
+If you need to customize the configuration, please refer to the comments in our [docker-compose.yml](docker/docker-compose.yaml) file and manually set the environment configuration. After making the changes, please run `docker-compose up -d` again. You can see the full list of environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
 If you'd like to configure a highly-available setup, there are community-contributed [Helm Charts](https://helm.sh/) and YAML files which allow Dify to be deployed on Kubernetes.
 
@@ -192,11 +191,6 @@ If you'd like to configure a highly-available setup, there are community-contrib
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
-#### Using Terraform for Deployment
-
-##### Azure Global
-Deploy Dify to Azure with a single click using [terraform](https://www.terraform.io/).
-- [Azure Terraform by @nikawang](https://github.com/nikawang/dify-azure-terraform)
 
 ## Contributing
 

README_AR.md

@@ -157,17 +157,15 @@
 
 ```bash
 cd docker
-cp .env.example .env
 docker compose up -d
 ```
-
 بعد التشغيل، يمكنك الوصول إلى لوحة تحكم Dify في متصفحك على [http://localhost/install](http://localhost/install) وبدء عملية التهيئة.
 
 > إذا كنت ترغب في المساهمة في Dify أو القيام بتطوير إضافي، فانظر إلى [دليلنا للنشر من الشفرة (code) المصدرية](https://docs.dify.ai/getting-started/install-self-hosted/local-source-code)
 
 ## الخطوات التالية
 
-إذا كنت بحاجة إلى تخصيص الإعدادات، فيرجى الرجوع إلى التعليقات في ملف [.env.example](docker/.env.example) وتحديث القيم المقابلة في ملف `.env`. بالإضافة إلى ذلك، قد تحتاج إلى إجراء تعديلات على ملف `docker-compose.yaml` نفسه، مثل تغيير إصدارات الصور أو تعيينات المنافذ أو نقاط تحميل وحدات التخزين، بناءً على بيئة النشر ومتطلباتك الخاصة. بعد إجراء أي تغييرات، يرجى إعادة تشغيل `docker-compose up -d`. يمكنك العثور على قائمة كاملة بمتغيرات البيئة المتاحة [هنا](https://docs.dify.ai/getting-started/install-self-hosted/environments).
+إذا كنت بحاجة إلى تخصيص التكوين، يرجى الرجوع إلى التعليقات في ملف [docker-compose.yml](docker/docker-compose.yaml) لدينا وتعيين التكوينات البيئية يدويًا. بعد إجراء التغييرات، يرجى تشغيل `docker-compose up -d` مرة أخرى. يمكنك رؤية قائمة كاملة بالمتغيرات البيئية [هنا](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
 يوجد مجتمع خاص بـ [Helm Charts](https://helm.sh/) وملفات YAML التي تسمح بتنفيذ Dify على Kubernetes للنظام من الإيجابيات العلوية.
 
@@ -175,12 +173,6 @@ docker compose up -d
 - [رسم بياني Helm من قبل @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [ملف YAML من قبل @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
-#### استخدام Terraform للتوزيع
-
-##### Azure Global
-استخدم [terraform](https://www.terraform.io/) لنشر Dify على Azure بنقرة واحدة.
-- [Azure Terraform بواسطة @nikawang](https://github.com/nikawang/dify-azure-terraform)
-
 
 ## المساهمة
 

README_CN.md

@@ -179,16 +179,11 @@ Dify 是一个开源的 LLM 应用开发平台。其直观的界面结合了 AI
 
 ```bash
 cd docker
-cp .env.example .env
 docker compose up -d
 ```
 
 运行后，可以在浏览器上访问 [http://localhost/install](http://localhost/install) 进入 Dify 控制台并开始初始化安装操作。
 
-### 自定义配置
-
-如果您需要自定义配置，请参考 [.env.example](docker/.env.example) 文件中的注释，并更新 `.env` 文件中对应的值。此外，您可能需要根据您的具体部署环境和需求对 `docker-compose.yaml` 文件本身进行调整，例如更改镜像版本、端口映射或卷挂载。完成任何更改后，请重新运行 `docker-compose up -d`。您可以在[此处](https://docs.dify.ai/getting-started/install-self-hosted/environments)找到可用环境变量的完整列表。
-
 #### 使用 Helm Chart 部署
 
 使用 [Helm Chart](https://helm.sh/) 版本或者 YAML 文件，可以在 Kubernetes 上部署 Dify。
@@ -197,11 +192,9 @@ docker compose up -d
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [YAML 文件 by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
-#### 使用 Terraform 部署
+### 配置
 
-##### Azure Global
-使用 [terraform](https://www.terraform.io/) 一键部署 Dify 到 Azure。
-- [Azure Terraform by @nikawang](https://github.com/nikawang/dify-azure-terraform)
+如果您需要自定义配置，请参考我们的 [docker-compose.yml](docker/docker-compose.yaml) 文件中的注释，并手动设置环境配置。更改后，请再次运行 `docker-compose up -d`。您可以在我们的[文档](https://docs.dify.ai/getting-started/install-self-hosted/environments)中查看所有环境变量的完整列表。
 
 ## Star History
 

README_ES.md

@@ -179,7 +179,6 @@ La forma más fácil de iniciar el servidor de Dify es ejecutar nuestro archivo
 
 ```bash
 cd docker
-cp .env.example .env
 docker compose up -d
 ```
 
@@ -189,7 +188,7 @@ Después de ejecutarlo, puedes acceder al panel de control de Dify en tu navegad
 
 ## Próximos pasos
 
-Si necesita personalizar la configuración, consulte los comentarios en nuestro archivo [.env.example](docker/.env.example) y actualice los valores correspondientes en su archivo `.env`. Además, es posible que deba realizar ajustes en el propio archivo `docker-compose.yaml`, como cambiar las versiones de las imágenes, las asignaciones de puertos o los montajes de volúmenes, según su entorno de implementación y requisitos específicos. Después de realizar cualquier cambio, vuelva a ejecutar `docker-compose up -d`. Puede encontrar la lista completa de variables de entorno disponibles [aquí](https://docs.dify.ai/getting-started/install-self-hosted/environments).
+Si necesitas personalizar la configuración, consulta los comentarios en nuestro archivo [docker-compose.yml](docker/docker-compose.yaml) y configura manualmente la configuración del entorno
 
 . Después de realizar los cambios, ejecuta `docker-compose up -d` nuevamente. Puedes ver la lista completa de variables de entorno [aquí](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
@@ -199,12 +198,6 @@ Si desea configurar una configuración de alta disponibilidad, la comunidad prop
 - [Gráfico Helm por @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [Ficheros YAML por @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
-#### Uso de Terraform para el despliegue
-
-##### Azure Global
-Utiliza [terraform](https://www.terraform.io/) para desplegar Dify en Azure con un solo clic.
-- [Azure Terraform por @nikawang](https://github.com/nikawang/dify-azure-terraform)
-
 
 ## Contribuir
 

README_FR.md

@@ -179,7 +179,6 @@ La manière la plus simple de démarrer le serveur Dify est d'exécuter notre fi
 
 ```bash
 cd docker
-cp .env.example .env
 docker compose up -d
 ```
 
@@ -189,7 +188,9 @@ Après l'exécution, vous pouvez accéder au tableau de bord Dify dans votre nav
 
 ## Prochaines étapes
 
-Si vous devez personnaliser la configuration, veuillez vous référer aux commentaires dans notre fichier [.env.example](docker/.env.example) et mettre à jour les valeurs correspondantes dans votre fichier `.env`. De plus, vous devrez peut-être apporter des modifications au fichier `docker-compose.yaml` lui-même, comme changer les versions d'image, les mappages de ports ou les montages de volumes, en fonction de votre environnement de déploiement et de vos exigences spécifiques. Après avoir effectué des modifications, veuillez réexécuter `docker-compose up -d`. Vous pouvez trouver la liste complète des variables d'environnement disponibles [ici](https://docs.dify.ai/getting-started/install-self-hosted/environments).
+Si vous devez personnaliser la configuration, veuillez
+
+ vous référer aux commentaires dans notre fichier [docker-compose.yml](docker/docker-compose.yaml) et définir manuellement la configuration de l'environnement. Après avoir apporté les modifications, veuillez exécuter à nouveau `docker-compose up -d`. Vous pouvez voir la liste complète des variables d'environnement [ici](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
 Si vous souhaitez configurer une configuration haute disponibilité, la communauté fournit des [Helm Charts](https://helm.sh/) et des fichiers YAML, à travers lesquels vous pouvez déployer Dify sur Kubernetes.
 
@@ -197,12 +198,6 @@ Si vous souhaitez configurer une configuration haute disponibilité, la communau
 - [Helm Chart par @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [Fichier YAML par @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
-#### Utilisation de Terraform pour le déploiement
-
-##### Azure Global
-Utilisez [terraform](https://www.terraform.io/) pour déployer Dify sur Azure en un clic.
-- [Azure Terraform par @nikawang](https://github.com/nikawang/dify-azure-terraform)
-
 
 ## Contribuer
 

README_JA.md

@@ -178,7 +178,6 @@ Difyサーバーを起動する最も簡単な方法は、[docker-compose.yml](d
 
 ```bash
 cd docker
-cp .env.example .env
 docker compose up -d
 ```
 
@@ -188,7 +187,7 @@ docker compose up -d
 
 ## 次のステップ
 
-設定をカスタマイズする必要がある場合は、[.env.example](docker/.env.example) ファイルのコメントを参照し、`.env` ファイルの対応する値を更新してください。さらに、デプロイ環境や要件に応じて、`docker-compose.yaml` ファイル自体を調整する必要がある場合があります。たとえば、イメージのバージョン、ポートのマッピング、ボリュームのマウントなどを変更します。変更を加えた後は、`docker-compose up -d` を再実行してください。利用可能な環境変数の全一覧は、[こちら](https://docs.dify.ai/getting-started/install-self-hosted/environments)で確認できます。
+環境設定をカスタマイズする場合は、[docker-compose.yml](docker/docker-compose.yaml)ファイル内のコメントを参照して、環境設定を手動で設定してください。変更を加えた後は、再び `docker-compose up -d` を実行してください。環境変数の完全なリストは[こちら](https://docs.dify.ai/getting-started/install-self-hosted/environments)をご覧ください。
 
 高可用性設定を設定する必要がある場合、コミュニティは[Helm Charts](https://helm.sh/)とYAMLファイルにより、DifyをKubernetesにデプロイすることができます。
 
@@ -196,12 +195,6 @@ docker compose up -d
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
-#### Terraformを使用したデプロイ
-
-##### Azure Global
-[terraform](https://www.terraform.io/) を使用して、AzureにDifyをワンクリックでデプロイします。
-- [nikawangのAzure Terraform](https://github.com/nikawang/dify-azure-terraform)
-
 
 ## 貢献
 

README_KL.md

@@ -179,7 +179,6 @@ The easiest way to start the Dify server is to run our [docker-compose.yml](dock
 
 ```bash
 cd docker
-cp .env.example .env
 docker compose up -d
 ```
 
@@ -189,7 +188,7 @@ After running, you can access the Dify dashboard in your browser at [http://loca
 
 ## Next steps
 
-If you need to customize the configuration, please refer to the comments in our [.env.example](docker/.env.example) file and update the corresponding values in your `.env` file. Additionally, you might need to make adjustments to the `docker-compose.yaml` file itself, such as changing image versions, port mappings, or volume mounts, based on your specific deployment environment and requirements. After making any changes, please re-run `docker-compose up -d`. You can find the full list of available environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
+If you need to customize the configuration, please refer to the comments in our [docker-compose.yml](docker/docker-compose.yaml) file and manually set the environment configuration. After making the changes, please run `docker-compose up -d` again. You can see the full list of environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
 If you'd like to configure a highly-available setup, there are community-contributed [Helm Charts](https://helm.sh/) and YAML files which allow Dify to be deployed on Kubernetes.
 
@@ -197,13 +196,6 @@ If you'd like to configure a highly-available setup, there are community-contrib
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
-#### Terraform atorlugu pilersitsineq
-
-##### Azure Global
-Atoruk [terraform](https://www.terraform.io/) Dify-mik Azure-mut ataatsikkut ikkussuilluarlugu.
-- [Azure Terraform atorlugu @nikawang](https://github.com/nikawang/dify-azure-terraform)
-
-
 ## Contributing
 
 For those who'd like to contribute code, see our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). 

README_KR.md

@@ -172,7 +172,6 @@ Dify 서버를 시작하는 가장 쉬운 방법은 [docker-compose.yml](docker/
 
 ```bash
 cd docker
-cp .env.example .env
 docker compose up -d
 ```
 
@@ -182,7 +181,8 @@ docker compose up -d
 
 ## 다음 단계
 
-구성을 사용자 정의해야 하는 경우 [.env.example](docker/.env.example) 파일의 주석을 참조하고 `.env` 파일에서 해당 값을 업데이트하십시오. 또한 특정 배포 환경 및 요구 사항에 따라 `docker-compose.yaml` 파일 자체를 조정해야 할 수도 있습니다. 예를 들어 이미지 버전, 포트 매핑 또는 볼륨 마운트를 변경합니다. 변경 한 후 `docker-compose up -d`를 다시 실행하십시오. 사용 가능한 환경 변수의 전체 목록은 [여기](https://docs.dify.ai/getting-started/install-self-hosted/environments)에서 찾을 수 있습니다.
+구성 커스터마이징이 필요한 경우, [docker-compose.yml](docker/docker-compose.yaml) 파일의 코멘트를 참조하여 환경 구성을 수동으로 설정하십시오. 변경 후 `docker-compose up -d` 를 다시 실행하십시오. 환경 변수의 전체 목록은 [여기](https://docs.dify.ai/getting-started/install-self-hosted/environments)에서 확인할 수 있습니다.
+
 
 Dify를 Kubernetes에 배포하고 프리미엄 스케일링 설정을 구성했다는 커뮤니티가 제공하는 [Helm Charts](https://helm.sh/)와 YAML 파일이 존재합니다.
 
@@ -190,12 +190,6 @@ Dify를 Kubernetes에 배포하고 프리미엄 스케일링 설정을 구성했
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
-#### Terraform을 사용한 배포
-
-##### Azure Global
-[terraform](https://www.terraform.io/)을 사용하여 Azure에 Dify를 원클릭으로 배포하세요.
-- [nikawang의 Azure Terraform](https://github.com/nikawang/dify-azure-terraform)
-
 ## 기여
 
 코드에 기여하고 싶은 분들은 [기여 가이드](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md)를 참조하세요.

api/.dockerignore

@@ -8,7 +8,4 @@ logs
 *.log*
 
 # jetbrains
-.idea
-
-# venv
-.venv
+.idea

api/.env.example

@@ -39,7 +39,7 @@ DB_DATABASE=dify
 
 # Storage configuration
 # use for store upload files, private keys...
-# storage type: local, s3, azure-blob, google-storage
+# storage type: local, s3, azure-blob
 STORAGE_TYPE=local
 STORAGE_LOCAL_PATH=storage
 S3_USE_AWS_MANAGED_IAM=false
@@ -63,7 +63,7 @@ ALIYUN_OSS_REGION=your-region
 
 # Google Storage configuration
 GOOGLE_STORAGE_BUCKET_NAME=yout-bucket-name
-GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64=your-google-service-account-json-base64-string
+GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON=your-google-service-account-json-base64-string
 
 # Tencent COS Storage configuration
 TENCENT_COS_BUCKET_NAME=your-bucket-name
@@ -72,18 +72,11 @@ TENCENT_COS_SECRET_ID=your-secret-id
 TENCENT_COS_REGION=your-region
 TENCENT_COS_SCHEME=your-scheme
 
-# OCI Storage configuration
-OCI_ENDPOINT=your-endpoint
-OCI_BUCKET_NAME=your-bucket-name
-OCI_ACCESS_KEY=your-access-key
-OCI_SECRET_KEY=your-secret-key
-OCI_REGION=your-region
-
 # CORS configuration
 WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 
-# Vector database configuration, support: weaviate, qdrant, milvus, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector
+# Vector database configuration, support: weaviate, qdrant, milvus, relyt, pgvecto_rs, pgvector
 VECTOR_STORE=weaviate
 
 # Weaviate configuration
@@ -151,13 +144,6 @@ CHROMA_DATABASE=default_database
 CHROMA_AUTH_PROVIDER=chromadb.auth.token_authn.TokenAuthenticationServerProvider
 CHROMA_AUTH_CREDENTIALS=difyai123456
 
-# OpenSearch configuration
-OPENSEARCH_HOST=127.0.0.1
-OPENSEARCH_PORT=9200
-OPENSEARCH_USER=admin
-OPENSEARCH_PASSWORD=admin
-OPENSEARCH_SECURE=true
-
 # Upload configuration
 UPLOAD_FILE_SIZE_LIMIT=15
 UPLOAD_FILE_BATCH_LIMIT=5

api/.vscode/launch.json

@@ -1,16 +1,30 @@
 {
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "Python: Celery",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "celery",
+            "justMyCode": true,
+            "args": ["-A", "app.celery", "worker", "-P", "gevent", "-c", "1", "--loglevel", "info", "-Q", "dataset,generation,mail"],
+            "envFile": "${workspaceFolder}/.env",
+            "env": {
+                "FLASK_APP": "app.py",
+                "FLASK_DEBUG": "1",
+                "GEVENT_SUPPORT": "True"
+            },
+            "console": "integratedTerminal",
+            "python": "${command:python.interpreterPath}"
+        },
         {
             "name": "Python: Flask",
             "type": "debugpy",
             "request": "launch",
-            "python": "${workspaceFolder}/api/.venv/bin/python",
-            "cwd": "${workspaceFolder}/api",
-            "envFile": ".env",
             "module": "flask",
-            "justMyCode": true,
-            "jinja": true,
             "env": {
                 "FLASK_APP": "app.py",
                 "FLASK_DEBUG": "1",
@@ -20,36 +34,11 @@
                 "run",
                 "--host=0.0.0.0",
                 "--port=5001",
-            ]
-        },
-        {
-            "name": "Python: Celery",
-            "type": "debugpy",
-            "request": "launch",
-            "python": "${workspaceFolder}/api/.venv/bin/python",
-            "cwd": "${workspaceFolder}/api",
-            "module": "celery",
+                "--debug"
+            ],
+            "jinja": true,
             "justMyCode": true,
-            "envFile": ".env",
-            "console": "integratedTerminal",
-            "env": {
-                "FLASK_APP": "app.py",
-                "FLASK_DEBUG": "1",
-                "GEVENT_SUPPORT": "True"
-            },
-            "args": [
-                "-A",
-                "app.celery",
-                "worker",
-                "-P",
-                "gevent",
-                "-c",
-                "1",
-                "--loglevel",
-                "info",
-                "-Q",
-                "dataset,generation,mail,ops_trace,app_deletion"
-            ]
-        },
+            "python": "${command:python.interpreterPath}"
+        }
     ]
 }

api/Dockerfile

@@ -1,43 +1,34 @@
 # base image
 FROM python:3.10-slim-bookworm AS base
 
-WORKDIR /app/api
+LABEL maintainer="takatost@gmail.com"
 
-# Install Poetry
-ENV POETRY_VERSION=1.8.3
-RUN pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir --upgrade poetry==${POETRY_VERSION}
-
-# Configure Poetry
-ENV POETRY_CACHE_DIR=/tmp/poetry_cache
-ENV POETRY_NO_INTERACTION=1
-ENV POETRY_VIRTUALENVS_IN_PROJECT=true
-ENV POETRY_VIRTUALENVS_CREATE=true
-
-FROM base AS packages
+# install packages
+FROM base as packages
 
 RUN apt-get update \
     && apt-get install -y --no-install-recommends gcc g++ libc-dev libffi-dev libgmp-dev libmpfr-dev libmpc-dev
 
-# Install Python dependencies
-COPY pyproject.toml poetry.lock ./
-RUN poetry install --sync --no-cache --no-root
+COPY requirements.txt /requirements.txt
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install --prefix=/pkg -r requirements.txt
 
 # production stage
 FROM base AS production
 
-ENV FLASK_APP=app.py
-ENV EDITION=SELF_HOSTED
-ENV DEPLOY_ENV=PRODUCTION
-ENV CONSOLE_API_URL=http://127.0.0.1:5001
-ENV CONSOLE_WEB_URL=http://127.0.0.1:3000
-ENV SERVICE_API_URL=http://127.0.0.1:5001
-ENV APP_WEB_URL=http://127.0.0.1:3000
+ENV FLASK_APP app.py
+ENV EDITION SELF_HOSTED
+ENV DEPLOY_ENV PRODUCTION
+ENV CONSOLE_API_URL http://127.0.0.1:5001
+ENV CONSOLE_WEB_URL http://127.0.0.1:3000
+ENV SERVICE_API_URL http://127.0.0.1:5001
+ENV APP_WEB_URL http://127.0.0.1:3000
 
 EXPOSE 5001
 
 # set timezone
-ENV TZ=UTC
+ENV TZ UTC
 
 WORKDIR /app/api
 
@@ -46,20 +37,13 @@ RUN apt-get update \
     && apt-get autoremove \
     && rm -rf /var/lib/apt/lists/*
 
-# Copy Python environment and packages
-ENV VIRTUAL_ENV=/app/api/.venv
-COPY --from=packages ${VIRTUAL_ENV} ${VIRTUAL_ENV}
-ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
-
-# Copy source code
+COPY --from=packages /pkg /usr/local
 COPY . /app/api/
 
-# Copy entrypoint
 COPY docker/entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 
-
 ARG COMMIT_SHA
-ENV COMMIT_SHA=${COMMIT_SHA}
+ENV COMMIT_SHA ${COMMIT_SHA}
 
-ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]

api/README.md

@@ -2,16 +2,13 @@
 
 ## Usage
 
-> [!IMPORTANT]
-> In the v0.6.12 release, we deprecated `pip` as the package management tool for Dify API Backend service and replaced it with `poetry`.
-
 1. Start the docker-compose stack
 
    The backend require some middleware, including PostgreSQL, Redis, and Weaviate, which can be started together using `docker-compose`.
 
    ```bash
    cd ../docker
-   docker compose -f docker-compose.middleware.yaml -p dify up -d
+   docker-compose -f docker-compose.middleware.yaml -p dify up -d
    cd ../api
    ```
 
@@ -32,8 +29,11 @@
 
    Dify API service uses [Poetry](https://python-poetry.org/docs/) to manage dependencies. You can execute `poetry shell` to activate the environment.
 
+   > Using pip can be found [below](#usage-with-pip).
+
 5. Install dependencies
 
+=======
    ```bash
    poetry env use 3.10
    poetry install
@@ -66,7 +66,7 @@
 10. If you need to debug local async processing, please start the worker service.
 
    ```bash
-   poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion
+   poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail
    ```
 
    The started celery app handles the async tasks, e.g. dataset importing and documents indexing.
@@ -85,3 +85,63 @@
    cd ../
    poetry run -C api bash dev/pytest/pytest_all_tests.sh
    ```
+
+## Usage with pip
+
+> [!NOTE]  
+> In the next version, we will deprecate pip as the primary package management tool for dify api service, currently Poetry and pip coexist.
+
+1. Start the docker-compose stack
+
+   The backend require some middleware, including PostgreSQL, Redis, and Weaviate, which can be started together using `docker-compose`.
+
+   ```bash
+   cd ../docker
+   docker-compose -f docker-compose.middleware.yaml -p dify up -d
+   cd ../api
+   ```
+
+2. Copy `.env.example` to `.env`
+3. Generate a `SECRET_KEY` in the `.env` file.
+
+   ```bash
+   sed -i "/^SECRET_KEY=/c\SECRET_KEY=$(openssl rand -base64 42)" .env
+   ```
+
+4. Create environment.
+
+   If you use Anaconda, create a new environment and activate it
+  
+   ```bash
+   conda create --name dify python=3.10
+   conda activate dify
+   ```
+
+5. Install dependencies
+
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+6. Run migrate
+
+   Before the first launch, migrate the database to the latest version.
+
+   ```bash
+   flask db upgrade
+   ```
+
+7. Start backend:
+
+   ```bash
+   flask run --host 0.0.0.0 --port=5001 --debug
+   ```
+
+8. Setup your application by visiting <http://localhost:5001/console/api/setup> or other apis...
+9. If you need to debug local async processing, please start the worker service.
+
+   ```bash
+   celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail
+   ```
+
+   The started celery app handles the async tasks, e.g. dataset importing and documents indexing.

api/app.py

@@ -1,8 +1,6 @@
 import os
 
-from configs.app_config import DifyConfig
-
-if not os.environ.get("DEBUG") or os.environ.get("DEBUG", "false").lower() != 'true':
+if not os.environ.get("DEBUG") or os.environ.get("DEBUG").lower() != 'true':
     from gevent import monkey
 
     monkey.patch_all()
@@ -24,6 +22,7 @@ from flask_cors import CORS
 from werkzeug.exceptions import Unauthorized
 
 from commands import register_commands
+from config import Config
 
 # DO NOT REMOVE BELOW
 from events import event_handlers
@@ -75,28 +74,10 @@ config_type = os.getenv('EDITION', default='SELF_HOSTED')  # ce edition first
 # Application Factory Function
 # ----------------------------
 
-def create_flask_app_with_configs() -> Flask:
-    """
-    create a raw flask app
-    with configs loaded from .env file
-    """
-    dify_app = DifyApp(__name__)
-    dify_app.config.from_mapping(DifyConfig().model_dump())
-
-    # populate configs into system environment variables
-    for key, value in dify_app.config.items():
-        if isinstance(value, str):
-            os.environ[key] = value
-        elif isinstance(value, int | float | bool):
-            os.environ[key] = str(value)
-        elif value is None:
-            os.environ[key] = ''
-
-    return dify_app
-
 
 def create_app() -> Flask:
-    app = create_flask_app_with_configs()
+    app = DifyApp(__name__)
+    app.config.from_object(Config())
 
     app.secret_key = app.config['SECRET_KEY']
 
@@ -118,21 +99,9 @@ def create_app() -> Flask:
         level=app.config.get('LOG_LEVEL'),
         format=app.config.get('LOG_FORMAT'),
         datefmt=app.config.get('LOG_DATEFORMAT'),
-        handlers=log_handlers,
-        force=True
+        handlers=log_handlers
     )
-    log_tz = app.config.get('LOG_TZ')
-    if log_tz:
-        from datetime import datetime
 
-        import pytz
-        timezone = pytz.timezone(log_tz)
-
-        def time_converter(seconds):
-            return datetime.utcfromtimestamp(seconds).astimezone(timezone).timetuple()
-
-        for handler in logging.root.handlers:
-            handler.formatter.converter = time_converter
     initialize_extensions(app)
     register_blueprints(app)
     register_commands(app)
@@ -160,26 +129,27 @@ def initialize_extensions(app):
 @login_manager.request_loader
 def load_user_from_request(request_from_flask_login):
     """Load user based on the request."""
-    if request.blueprint not in ['console', 'inner_api']:
-        return None
-    # Check if the user_id contains a dot, indicating the old format
-    auth_header = request.headers.get('Authorization', '')
-    if not auth_header:
-        auth_token = request.args.get('_token')
-        if not auth_token:
-            raise Unauthorized('Invalid Authorization token.')
+    if request.blueprint in ['console', 'inner_api']:
+        # Check if the user_id contains a dot, indicating the old format
+        auth_header = request.headers.get('Authorization', '')
+        if not auth_header:
+            auth_token = request.args.get('_token')
+            if not auth_token:
+                raise Unauthorized('Invalid Authorization token.')
+        else:
+            if ' ' not in auth_header:
+                raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.')
+            auth_scheme, auth_token = auth_header.split(None, 1)
+            auth_scheme = auth_scheme.lower()
+            if auth_scheme != 'bearer':
+                raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.')
+
+        decoded = PassportService().verify(auth_token)
+        user_id = decoded.get('user_id')
+
+        return AccountService.load_user(user_id)
     else:
-        if ' ' not in auth_header:
-            raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.')
-        auth_scheme, auth_token = auth_header.split(None, 1)
-        auth_scheme = auth_scheme.lower()
-        if auth_scheme != 'bearer':
-            raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.')
-
-    decoded = PassportService().verify(auth_token)
-    user_id = decoded.get('user_id')
-
-    return AccountService.load_logged_in_account(account_id=user_id, token=auth_token)
+        return None
 
 
 @login_manager.unauthorized_handler
@@ -240,7 +210,7 @@ def register_blueprints(app):
 app = create_app()
 celery = app.extensions["celery"]
 
-if app.config.get('TESTING'):
+if app.config['TESTING']:
     print("App is running in TESTING mode")
 
 

api/commands.py

@@ -12,7 +12,6 @@ from constants.languages import languages
 from core.rag.datasource.vdb.vector_factory import Vector
 from core.rag.datasource.vdb.vector_type import VectorType
 from core.rag.models.document import Document
-from events.app_event import app_was_created
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from libs.helper import email as email_validate
@@ -328,14 +327,6 @@ def migrate_knowledge_vector_database():
                         "vector_store": {"class_prefix": collection_name}
                     }
                     dataset.index_struct = json.dumps(index_struct_dict)
-                elif vector_type == VectorType.OPENSEARCH:
-                    dataset_id = dataset.id
-                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
-                    index_struct_dict = {
-                        "type": VectorType.OPENSEARCH,
-                        "vector_store": {"class_prefix": collection_name}
-                    }
-                    dataset.index_struct = json.dumps(index_struct_dict)
                 else:
                     raise ValueError(f"Vector store {vector_type} is not supported.")
 
@@ -586,53 +577,6 @@ def upgrade_db():
         click.echo('Database migration skipped')
 
 
-@click.command('fix-app-site-missing', help='Fix app related site missing issue.')
-def fix_app_site_missing():
-    """
-    Fix app related site missing issue.
-    """
-    click.echo(click.style('Start fix app related site missing issue.', fg='green'))
-
-    failed_app_ids = []
-    while True:
-        sql = """select apps.id as id from apps left join sites on sites.app_id=apps.id
-where sites.id is null limit 1000"""
-        with db.engine.begin() as conn:
-            rs = conn.execute(db.text(sql))
-
-            processed_count = 0
-            for i in rs:
-                processed_count += 1
-                app_id = str(i.id)
-
-                if app_id in failed_app_ids:
-                    continue
-
-                try:
-                    app = db.session.query(App).filter(App.id == app_id).first()
-                    tenant = app.tenant
-                    if tenant:
-                        accounts = tenant.get_accounts()
-                        if not accounts:
-                            print("Fix app {} failed.".format(app.id))
-                            continue
-
-                        account = accounts[0]
-                        print("Fix app {} related site missing issue.".format(app.id))
-                        app_was_created.send(app, account=account)
-                except Exception as e:
-                    failed_app_ids.append(app_id)
-                    click.echo(click.style('Fix app {} related site missing issue failed!'.format(app_id), fg='red'))
-                    logging.exception(f'Fix app related site missing issue failed, error: {e}')
-                    continue
-
-            if not processed_count:
-                break
-
-
-    click.echo(click.style('Congratulations! Fix app related site missing issue successful!', fg='green'))
-
-
 def register_commands(app):
     app.cli.add_command(reset_password)
     app.cli.add_command(reset_email)
@@ -642,4 +586,3 @@ def register_commands(app):
     app.cli.add_command(add_qdrant_doc_id_index)
     app.cli.add_command(create_tenant)
     app.cli.add_command(upgrade_db)
-    app.cli.add_command(fix_app_site_missing)

api/config.py

@@ -0,0 +1,458 @@
+import os
+
+import dotenv
+
+dotenv.load_dotenv()
+
+DEFAULTS = {
+    'EDITION': 'SELF_HOSTED',
+    'DB_USERNAME': 'postgres',
+    'DB_PASSWORD': '',
+    'DB_HOST': 'localhost',
+    'DB_PORT': '5432',
+    'DB_DATABASE': 'dify',
+    'DB_CHARSET': '',
+    'REDIS_HOST': 'localhost',
+    'REDIS_PORT': '6379',
+    'REDIS_DB': '0',
+    'REDIS_USE_SSL': 'False',
+    'OAUTH_REDIRECT_PATH': '/console/api/oauth/authorize',
+    'OAUTH_REDIRECT_INDEX_PATH': '/',
+    'CONSOLE_WEB_URL': 'https://cloud.dify.ai',
+    'CONSOLE_API_URL': 'https://cloud.dify.ai',
+    'SERVICE_API_URL': 'https://api.dify.ai',
+    'APP_WEB_URL': 'https://udify.app',
+    'FILES_URL': '',
+    'FILES_ACCESS_TIMEOUT': 300,
+    'S3_USE_AWS_MANAGED_IAM': 'False',
+    'S3_ADDRESS_STYLE': 'auto',
+    'STORAGE_TYPE': 'local',
+    'STORAGE_LOCAL_PATH': 'storage',
+    'CHECK_UPDATE_URL': 'https://updates.dify.ai',
+    'DEPLOY_ENV': 'PRODUCTION',
+    'SQLALCHEMY_DATABASE_URI_SCHEME': 'postgresql',
+    'SQLALCHEMY_POOL_SIZE': 30,
+    'SQLALCHEMY_MAX_OVERFLOW': 10,
+    'SQLALCHEMY_POOL_RECYCLE': 3600,
+    'SQLALCHEMY_POOL_PRE_PING': 'False',
+    'SQLALCHEMY_ECHO': 'False',
+    'SENTRY_TRACES_SAMPLE_RATE': 1.0,
+    'SENTRY_PROFILES_SAMPLE_RATE': 1.0,
+    'WEAVIATE_GRPC_ENABLED': 'True',
+    'WEAVIATE_BATCH_SIZE': 100,
+    'QDRANT_CLIENT_TIMEOUT': 20,
+    'QDRANT_GRPC_ENABLED': 'False',
+    'QDRANT_GRPC_PORT': '6334',
+    'CELERY_BACKEND': 'database',
+    'LOG_LEVEL': 'INFO',
+    'LOG_FILE': '',
+    'LOG_FORMAT': '%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s',
+    'LOG_DATEFORMAT': '%Y-%m-%d %H:%M:%S',
+    'HOSTED_OPENAI_QUOTA_LIMIT': 200,
+    'HOSTED_OPENAI_TRIAL_ENABLED': 'False',
+    'HOSTED_OPENAI_TRIAL_MODELS': 'gpt-3.5-turbo,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-16k,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-0613,gpt-3.5-turbo-0125,text-davinci-003',
+    'HOSTED_OPENAI_PAID_ENABLED': 'False',
+    'HOSTED_OPENAI_PAID_MODELS': 'gpt-4,gpt-4-turbo-preview,gpt-4-turbo-2024-04-09,gpt-4-1106-preview,gpt-4-0125-preview,gpt-3.5-turbo,gpt-3.5-turbo-16k,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-1106,gpt-3.5-turbo-0613,gpt-3.5-turbo-0125,gpt-3.5-turbo-instruct,text-davinci-003',
+    'HOSTED_AZURE_OPENAI_ENABLED': 'False',
+    'HOSTED_AZURE_OPENAI_QUOTA_LIMIT': 200,
+    'HOSTED_ANTHROPIC_QUOTA_LIMIT': 600000,
+    'HOSTED_ANTHROPIC_TRIAL_ENABLED': 'False',
+    'HOSTED_ANTHROPIC_PAID_ENABLED': 'False',
+    'HOSTED_MODERATION_ENABLED': 'False',
+    'HOSTED_MODERATION_PROVIDERS': '',
+    'HOSTED_FETCH_APP_TEMPLATES_MODE': 'remote',
+    'HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN': 'https://tmpl.dify.ai',
+    'CLEAN_DAY_SETTING': 30,
+    'UPLOAD_FILE_SIZE_LIMIT': 15,
+    'UPLOAD_FILE_BATCH_LIMIT': 5,
+    'UPLOAD_IMAGE_FILE_SIZE_LIMIT': 10,
+    'OUTPUT_MODERATION_BUFFER_SIZE': 300,
+    'MULTIMODAL_SEND_IMAGE_FORMAT': 'base64',
+    'INVITE_EXPIRY_HOURS': 72,
+    'BILLING_ENABLED': 'False',
+    'CAN_REPLACE_LOGO': 'False',
+    'MODEL_LB_ENABLED': 'False',
+    'ETL_TYPE': 'dify',
+    'KEYWORD_STORE': 'jieba',
+    'BATCH_UPLOAD_LIMIT': 20,
+    'CODE_EXECUTION_ENDPOINT': 'http://sandbox:8194',
+    'CODE_EXECUTION_API_KEY': 'dify-sandbox',
+    'TOOL_ICON_CACHE_MAX_AGE': 3600,
+    'MILVUS_DATABASE': 'default',
+    'KEYWORD_DATA_SOURCE_TYPE': 'database',
+    'INNER_API': 'False',
+    'ENTERPRISE_ENABLED': 'False',
+    'INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH': 1000,
+    'WORKFLOW_MAX_EXECUTION_STEPS': 500,
+    'WORKFLOW_MAX_EXECUTION_TIME': 1200,
+    'WORKFLOW_CALL_MAX_DEPTH': 5,
+    'APP_MAX_EXECUTION_TIME': 1200,
+}
+
+
+def get_env(key):
+    return os.environ.get(key, DEFAULTS.get(key))
+
+
+def get_bool_env(key):
+    value = get_env(key)
+    return value.lower() == 'true' if value is not None else False
+
+
+def get_cors_allow_origins(env, default):
+    cors_allow_origins = []
+    if get_env(env):
+        for origin in get_env(env).split(','):
+            cors_allow_origins.append(origin)
+    else:
+        cors_allow_origins = [default]
+
+    return cors_allow_origins
+
+
+class Config:
+    """Application configuration class."""
+
+    def __init__(self):
+        # ------------------------
+        # General Configurations.
+        # ------------------------
+        self.CURRENT_VERSION = "0.6.11"
+        self.COMMIT_SHA = get_env('COMMIT_SHA')
+        self.EDITION = get_env('EDITION')
+        self.DEPLOY_ENV = get_env('DEPLOY_ENV')
+        self.TESTING = False
+        self.LOG_LEVEL = get_env('LOG_LEVEL')
+        self.LOG_FILE = get_env('LOG_FILE')
+        self.LOG_FORMAT = get_env('LOG_FORMAT')
+        self.LOG_DATEFORMAT = get_env('LOG_DATEFORMAT')
+        self.API_COMPRESSION_ENABLED = get_bool_env('API_COMPRESSION_ENABLED')
+
+        # The backend URL prefix of the console API.
+        # used to concatenate the login authorization callback or notion integration callback.
+        self.CONSOLE_API_URL = get_env('CONSOLE_API_URL')
+
+        # The front-end URL prefix of the console web.
+        # used to concatenate some front-end addresses and for CORS configuration use.
+        self.CONSOLE_WEB_URL = get_env('CONSOLE_WEB_URL')
+
+        # WebApp Url prefix.
+        # used to display WebAPP API Base Url to the front-end.
+        self.APP_WEB_URL = get_env('APP_WEB_URL')
+
+        # Service API Url prefix.
+        # used to display Service API Base Url to the front-end.
+        self.SERVICE_API_URL = get_env('SERVICE_API_URL')
+
+        # File preview or download Url prefix.
+        # used to display File preview or download Url to the front-end or as Multi-model inputs;
+        # Url is signed and has expiration time.
+        self.FILES_URL = get_env('FILES_URL') if get_env('FILES_URL') else self.CONSOLE_API_URL
+
+        # File Access Time specifies a time interval in seconds for the file to be accessed.
+        # The default value is 300 seconds.
+        self.FILES_ACCESS_TIMEOUT = int(get_env('FILES_ACCESS_TIMEOUT'))
+
+        # Your App secret key will be used for securely signing the session cookie
+        # Make sure you are changing this key for your deployment with a strong key.
+        # You can generate a strong key using `openssl rand -base64 42`.
+        # Alternatively you can set it with `SECRET_KEY` environment variable.
+        self.SECRET_KEY = get_env('SECRET_KEY')
+
+        # Enable or disable the inner API.
+        self.INNER_API = get_bool_env('INNER_API')
+        # The inner API key is used to authenticate the inner API.
+        self.INNER_API_KEY = get_env('INNER_API_KEY')
+
+        # cors settings
+        self.CONSOLE_CORS_ALLOW_ORIGINS = get_cors_allow_origins(
+            'CONSOLE_CORS_ALLOW_ORIGINS', self.CONSOLE_WEB_URL)
+        self.WEB_API_CORS_ALLOW_ORIGINS = get_cors_allow_origins(
+            'WEB_API_CORS_ALLOW_ORIGINS', '*')
+
+        # check update url
+        self.CHECK_UPDATE_URL = get_env('CHECK_UPDATE_URL')
+
+        # ------------------------
+        # Database Configurations.
+        # ------------------------
+        db_credentials = {
+            key: get_env(key) for key in
+            ['DB_USERNAME', 'DB_PASSWORD', 'DB_HOST', 'DB_PORT', 'DB_DATABASE', 'DB_CHARSET']
+        }
+        self.SQLALCHEMY_DATABASE_URI_SCHEME = get_env('SQLALCHEMY_DATABASE_URI_SCHEME')
+
+        db_extras = f"?client_encoding={db_credentials['DB_CHARSET']}" if db_credentials['DB_CHARSET'] else ""
+
+        self.SQLALCHEMY_DATABASE_URI = f"{self.SQLALCHEMY_DATABASE_URI_SCHEME}://{db_credentials['DB_USERNAME']}:{db_credentials['DB_PASSWORD']}@{db_credentials['DB_HOST']}:{db_credentials['DB_PORT']}/{db_credentials['DB_DATABASE']}{db_extras}"
+        self.SQLALCHEMY_ENGINE_OPTIONS = {
+            'pool_size': int(get_env('SQLALCHEMY_POOL_SIZE')),
+            'max_overflow': int(get_env('SQLALCHEMY_MAX_OVERFLOW')),
+            'pool_recycle': int(get_env('SQLALCHEMY_POOL_RECYCLE')),
+            'pool_pre_ping': get_bool_env('SQLALCHEMY_POOL_PRE_PING'),
+            'connect_args': {'options': '-c timezone=UTC'},
+        }
+
+        self.SQLALCHEMY_ECHO = get_bool_env('SQLALCHEMY_ECHO')
+
+        # ------------------------
+        # Redis Configurations.
+        # ------------------------
+        self.REDIS_HOST = get_env('REDIS_HOST')
+        self.REDIS_PORT = get_env('REDIS_PORT')
+        self.REDIS_USERNAME = get_env('REDIS_USERNAME')
+        self.REDIS_PASSWORD = get_env('REDIS_PASSWORD')
+        self.REDIS_DB = get_env('REDIS_DB')
+        self.REDIS_USE_SSL = get_bool_env('REDIS_USE_SSL')
+
+        # ------------------------
+        # Celery worker Configurations.
+        # ------------------------
+        self.CELERY_BROKER_URL = get_env('CELERY_BROKER_URL')
+        self.CELERY_BACKEND = get_env('CELERY_BACKEND')
+        self.CELERY_RESULT_BACKEND = 'db+{}'.format(self.SQLALCHEMY_DATABASE_URI) \
+            if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL
+        self.BROKER_USE_SSL = self.CELERY_BROKER_URL.startswith('rediss://')
+
+        # ------------------------
+        # Code Execution Sandbox Configurations.
+        # ------------------------
+        self.CODE_EXECUTION_ENDPOINT = get_env('CODE_EXECUTION_ENDPOINT')
+        self.CODE_EXECUTION_API_KEY = get_env('CODE_EXECUTION_API_KEY')
+
+        # ------------------------
+        # File Storage Configurations.
+        # ------------------------
+        self.STORAGE_TYPE = get_env('STORAGE_TYPE')
+        self.STORAGE_LOCAL_PATH = get_env('STORAGE_LOCAL_PATH')
+
+        # S3 Storage settings
+        self.S3_USE_AWS_MANAGED_IAM = get_bool_env('S3_USE_AWS_MANAGED_IAM')
+        self.S3_ENDPOINT = get_env('S3_ENDPOINT')
+        self.S3_BUCKET_NAME = get_env('S3_BUCKET_NAME')
+        self.S3_ACCESS_KEY = get_env('S3_ACCESS_KEY')
+        self.S3_SECRET_KEY = get_env('S3_SECRET_KEY')
+        self.S3_REGION = get_env('S3_REGION')
+        self.S3_ADDRESS_STYLE = get_env('S3_ADDRESS_STYLE')
+
+        # Azure Blob Storage settings
+        self.AZURE_BLOB_ACCOUNT_NAME = get_env('AZURE_BLOB_ACCOUNT_NAME')
+        self.AZURE_BLOB_ACCOUNT_KEY = get_env('AZURE_BLOB_ACCOUNT_KEY')
+        self.AZURE_BLOB_CONTAINER_NAME = get_env('AZURE_BLOB_CONTAINER_NAME')
+        self.AZURE_BLOB_ACCOUNT_URL = get_env('AZURE_BLOB_ACCOUNT_URL')
+
+        # Aliyun Storage settings
+        self.ALIYUN_OSS_BUCKET_NAME = get_env('ALIYUN_OSS_BUCKET_NAME')
+        self.ALIYUN_OSS_ACCESS_KEY = get_env('ALIYUN_OSS_ACCESS_KEY')
+        self.ALIYUN_OSS_SECRET_KEY = get_env('ALIYUN_OSS_SECRET_KEY')
+        self.ALIYUN_OSS_ENDPOINT = get_env('ALIYUN_OSS_ENDPOINT')
+        self.ALIYUN_OSS_REGION = get_env('ALIYUN_OSS_REGION')
+        self.ALIYUN_OSS_AUTH_VERSION = get_env('ALIYUN_OSS_AUTH_VERSION')
+
+        # Google Cloud Storage settings
+        self.GOOGLE_STORAGE_BUCKET_NAME = get_env('GOOGLE_STORAGE_BUCKET_NAME')
+        self.GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64 = get_env('GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64')
+
+        # Tencent Cos Storage settings
+        self.TENCENT_COS_BUCKET_NAME = get_env('TENCENT_COS_BUCKET_NAME')
+        self.TENCENT_COS_REGION = get_env('TENCENT_COS_REGION')
+        self.TENCENT_COS_SECRET_ID = get_env('TENCENT_COS_SECRET_ID')
+        self.TENCENT_COS_SECRET_KEY = get_env('TENCENT_COS_SECRET_KEY')
+        self.TENCENT_COS_SCHEME = get_env('TENCENT_COS_SCHEME')
+
+        # ------------------------
+        # Vector Store Configurations.
+        # Currently, only support: qdrant, milvus, zilliz, weaviate, relyt, pgvector
+        # ------------------------
+        self.VECTOR_STORE = get_env('VECTOR_STORE')
+        self.KEYWORD_STORE = get_env('KEYWORD_STORE')
+
+        # qdrant settings
+        self.QDRANT_URL = get_env('QDRANT_URL')
+        self.QDRANT_API_KEY = get_env('QDRANT_API_KEY')
+        self.QDRANT_CLIENT_TIMEOUT = get_env('QDRANT_CLIENT_TIMEOUT')
+        self.QDRANT_GRPC_ENABLED = get_env('QDRANT_GRPC_ENABLED')
+        self.QDRANT_GRPC_PORT = get_env('QDRANT_GRPC_PORT')
+
+        # milvus / zilliz setting
+        self.MILVUS_HOST = get_env('MILVUS_HOST')
+        self.MILVUS_PORT = get_env('MILVUS_PORT')
+        self.MILVUS_USER = get_env('MILVUS_USER')
+        self.MILVUS_PASSWORD = get_env('MILVUS_PASSWORD')
+        self.MILVUS_SECURE = get_env('MILVUS_SECURE')
+        self.MILVUS_DATABASE = get_env('MILVUS_DATABASE')
+
+        # weaviate settings
+        self.WEAVIATE_ENDPOINT = get_env('WEAVIATE_ENDPOINT')
+        self.WEAVIATE_API_KEY = get_env('WEAVIATE_API_KEY')
+        self.WEAVIATE_GRPC_ENABLED = get_bool_env('WEAVIATE_GRPC_ENABLED')
+        self.WEAVIATE_BATCH_SIZE = int(get_env('WEAVIATE_BATCH_SIZE'))
+
+        # relyt settings
+        self.RELYT_HOST = get_env('RELYT_HOST')
+        self.RELYT_PORT = get_env('RELYT_PORT')
+        self.RELYT_USER = get_env('RELYT_USER')
+        self.RELYT_PASSWORD = get_env('RELYT_PASSWORD')
+        self.RELYT_DATABASE = get_env('RELYT_DATABASE')
+
+
+        # tencent settings
+        self.TENCENT_VECTOR_DB_URL = get_env('TENCENT_VECTOR_DB_URL')
+        self.TENCENT_VECTOR_DB_API_KEY = get_env('TENCENT_VECTOR_DB_API_KEY')
+        self.TENCENT_VECTOR_DB_TIMEOUT = get_env('TENCENT_VECTOR_DB_TIMEOUT')
+        self.TENCENT_VECTOR_DB_USERNAME = get_env('TENCENT_VECTOR_DB_USERNAME')
+        self.TENCENT_VECTOR_DB_DATABASE = get_env('TENCENT_VECTOR_DB_DATABASE')
+        self.TENCENT_VECTOR_DB_SHARD = get_env('TENCENT_VECTOR_DB_SHARD')
+        self.TENCENT_VECTOR_DB_REPLICAS = get_env('TENCENT_VECTOR_DB_REPLICAS')
+
+        # pgvecto rs settings
+        self.PGVECTO_RS_HOST = get_env('PGVECTO_RS_HOST')
+        self.PGVECTO_RS_PORT = get_env('PGVECTO_RS_PORT')
+        self.PGVECTO_RS_USER = get_env('PGVECTO_RS_USER')
+        self.PGVECTO_RS_PASSWORD = get_env('PGVECTO_RS_PASSWORD')
+        self.PGVECTO_RS_DATABASE = get_env('PGVECTO_RS_DATABASE')
+
+        # pgvector settings
+        self.PGVECTOR_HOST = get_env('PGVECTOR_HOST')
+        self.PGVECTOR_PORT = get_env('PGVECTOR_PORT')
+        self.PGVECTOR_USER = get_env('PGVECTOR_USER')
+        self.PGVECTOR_PASSWORD = get_env('PGVECTOR_PASSWORD')
+        self.PGVECTOR_DATABASE = get_env('PGVECTOR_DATABASE')
+
+        # tidb-vector settings
+        self.TIDB_VECTOR_HOST = get_env('TIDB_VECTOR_HOST')
+        self.TIDB_VECTOR_PORT = get_env('TIDB_VECTOR_PORT')
+        self.TIDB_VECTOR_USER = get_env('TIDB_VECTOR_USER')
+        self.TIDB_VECTOR_PASSWORD = get_env('TIDB_VECTOR_PASSWORD')
+        self.TIDB_VECTOR_DATABASE = get_env('TIDB_VECTOR_DATABASE')
+
+        # chroma settings
+        self.CHROMA_HOST = get_env('CHROMA_HOST')
+        self.CHROMA_PORT = get_env('CHROMA_PORT')
+        self.CHROMA_TENANT = get_env('CHROMA_TENANT')
+        self.CHROMA_DATABASE = get_env('CHROMA_DATABASE')
+        self.CHROMA_AUTH_PROVIDER = get_env('CHROMA_AUTH_PROVIDER')
+        self.CHROMA_AUTH_CREDENTIALS = get_env('CHROMA_AUTH_CREDENTIALS')
+
+        # ------------------------
+        # Mail Configurations.
+        # ------------------------
+        self.MAIL_TYPE = get_env('MAIL_TYPE')
+        self.MAIL_DEFAULT_SEND_FROM = get_env('MAIL_DEFAULT_SEND_FROM')
+        self.RESEND_API_KEY = get_env('RESEND_API_KEY')
+        self.RESEND_API_URL = get_env('RESEND_API_URL')
+        # SMTP settings
+        self.SMTP_SERVER = get_env('SMTP_SERVER')
+        self.SMTP_PORT = get_env('SMTP_PORT')
+        self.SMTP_USERNAME = get_env('SMTP_USERNAME')
+        self.SMTP_PASSWORD = get_env('SMTP_PASSWORD')
+        self.SMTP_USE_TLS = get_bool_env('SMTP_USE_TLS')
+        self.SMTP_OPPORTUNISTIC_TLS = get_bool_env('SMTP_OPPORTUNISTIC_TLS')
+
+        # ------------------------
+        # Workspace Configurations.
+        # ------------------------
+        self.INVITE_EXPIRY_HOURS = int(get_env('INVITE_EXPIRY_HOURS'))
+
+        # ------------------------
+        # Sentry Configurations.
+        # ------------------------
+        self.SENTRY_DSN = get_env('SENTRY_DSN')
+        self.SENTRY_TRACES_SAMPLE_RATE = float(get_env('SENTRY_TRACES_SAMPLE_RATE'))
+        self.SENTRY_PROFILES_SAMPLE_RATE = float(get_env('SENTRY_PROFILES_SAMPLE_RATE'))
+
+        # ------------------------
+        # Business Configurations.
+        # ------------------------
+
+        # multi model send image format, support base64, url, default is base64
+        self.MULTIMODAL_SEND_IMAGE_FORMAT = get_env('MULTIMODAL_SEND_IMAGE_FORMAT')
+
+        # Dataset Configurations.
+        self.CLEAN_DAY_SETTING = get_env('CLEAN_DAY_SETTING')
+
+        # File upload Configurations.
+        self.UPLOAD_FILE_SIZE_LIMIT = int(get_env('UPLOAD_FILE_SIZE_LIMIT'))
+        self.UPLOAD_FILE_BATCH_LIMIT = int(get_env('UPLOAD_FILE_BATCH_LIMIT'))
+        self.UPLOAD_IMAGE_FILE_SIZE_LIMIT = int(get_env('UPLOAD_IMAGE_FILE_SIZE_LIMIT'))
+        self.BATCH_UPLOAD_LIMIT = get_env('BATCH_UPLOAD_LIMIT')
+
+        # RAG ETL Configurations.
+        self.ETL_TYPE = get_env('ETL_TYPE')
+        self.UNSTRUCTURED_API_URL = get_env('UNSTRUCTURED_API_URL')
+        self.UNSTRUCTURED_API_KEY = get_env('UNSTRUCTURED_API_KEY')
+        self.KEYWORD_DATA_SOURCE_TYPE = get_env('KEYWORD_DATA_SOURCE_TYPE')
+
+        # Indexing Configurations.
+        self.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH = get_env('INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH')
+
+        # Tool Configurations.
+        self.TOOL_ICON_CACHE_MAX_AGE = get_env('TOOL_ICON_CACHE_MAX_AGE')
+
+        self.WORKFLOW_MAX_EXECUTION_STEPS = int(get_env('WORKFLOW_MAX_EXECUTION_STEPS'))
+        self.WORKFLOW_MAX_EXECUTION_TIME = int(get_env('WORKFLOW_MAX_EXECUTION_TIME'))
+        self.WORKFLOW_CALL_MAX_DEPTH = int(get_env('WORKFLOW_CALL_MAX_DEPTH'))
+        self.APP_MAX_EXECUTION_TIME = int(get_env('APP_MAX_EXECUTION_TIME'))
+
+        # Moderation in app Configurations.
+        self.OUTPUT_MODERATION_BUFFER_SIZE = int(get_env('OUTPUT_MODERATION_BUFFER_SIZE'))
+
+        # Notion integration setting
+        self.NOTION_CLIENT_ID = get_env('NOTION_CLIENT_ID')
+        self.NOTION_CLIENT_SECRET = get_env('NOTION_CLIENT_SECRET')
+        self.NOTION_INTEGRATION_TYPE = get_env('NOTION_INTEGRATION_TYPE')
+        self.NOTION_INTERNAL_SECRET = get_env('NOTION_INTERNAL_SECRET')
+        self.NOTION_INTEGRATION_TOKEN = get_env('NOTION_INTEGRATION_TOKEN')
+
+        # ------------------------
+        # Platform Configurations.
+        # ------------------------
+        self.GITHUB_CLIENT_ID = get_env('GITHUB_CLIENT_ID')
+        self.GITHUB_CLIENT_SECRET = get_env('GITHUB_CLIENT_SECRET')
+        self.GOOGLE_CLIENT_ID = get_env('GOOGLE_CLIENT_ID')
+        self.GOOGLE_CLIENT_SECRET = get_env('GOOGLE_CLIENT_SECRET')
+        self.OAUTH_REDIRECT_PATH = get_env('OAUTH_REDIRECT_PATH')
+
+        self.HOSTED_OPENAI_API_KEY = get_env('HOSTED_OPENAI_API_KEY')
+        self.HOSTED_OPENAI_API_BASE = get_env('HOSTED_OPENAI_API_BASE')
+        self.HOSTED_OPENAI_API_ORGANIZATION = get_env('HOSTED_OPENAI_API_ORGANIZATION')
+        self.HOSTED_OPENAI_TRIAL_ENABLED = get_bool_env('HOSTED_OPENAI_TRIAL_ENABLED')
+        self.HOSTED_OPENAI_TRIAL_MODELS = get_env('HOSTED_OPENAI_TRIAL_MODELS')
+        self.HOSTED_OPENAI_QUOTA_LIMIT = int(get_env('HOSTED_OPENAI_QUOTA_LIMIT'))
+        self.HOSTED_OPENAI_PAID_ENABLED = get_bool_env('HOSTED_OPENAI_PAID_ENABLED')
+        self.HOSTED_OPENAI_PAID_MODELS = get_env('HOSTED_OPENAI_PAID_MODELS')
+
+        self.HOSTED_AZURE_OPENAI_ENABLED = get_bool_env('HOSTED_AZURE_OPENAI_ENABLED')
+        self.HOSTED_AZURE_OPENAI_API_KEY = get_env('HOSTED_AZURE_OPENAI_API_KEY')
+        self.HOSTED_AZURE_OPENAI_API_BASE = get_env('HOSTED_AZURE_OPENAI_API_BASE')
+        self.HOSTED_AZURE_OPENAI_QUOTA_LIMIT = int(get_env('HOSTED_AZURE_OPENAI_QUOTA_LIMIT'))
+
+        self.HOSTED_ANTHROPIC_API_BASE = get_env('HOSTED_ANTHROPIC_API_BASE')
+        self.HOSTED_ANTHROPIC_API_KEY = get_env('HOSTED_ANTHROPIC_API_KEY')
+        self.HOSTED_ANTHROPIC_TRIAL_ENABLED = get_bool_env('HOSTED_ANTHROPIC_TRIAL_ENABLED')
+        self.HOSTED_ANTHROPIC_QUOTA_LIMIT = int(get_env('HOSTED_ANTHROPIC_QUOTA_LIMIT'))
+        self.HOSTED_ANTHROPIC_PAID_ENABLED = get_bool_env('HOSTED_ANTHROPIC_PAID_ENABLED')
+
+        self.HOSTED_MINIMAX_ENABLED = get_bool_env('HOSTED_MINIMAX_ENABLED')
+        self.HOSTED_SPARK_ENABLED = get_bool_env('HOSTED_SPARK_ENABLED')
+        self.HOSTED_ZHIPUAI_ENABLED = get_bool_env('HOSTED_ZHIPUAI_ENABLED')
+
+        self.HOSTED_MODERATION_ENABLED = get_bool_env('HOSTED_MODERATION_ENABLED')
+        self.HOSTED_MODERATION_PROVIDERS = get_env('HOSTED_MODERATION_PROVIDERS')
+
+        # fetch app templates mode, remote, builtin, db(only for dify SaaS), default: remote
+        self.HOSTED_FETCH_APP_TEMPLATES_MODE = get_env('HOSTED_FETCH_APP_TEMPLATES_MODE')
+        self.HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN = get_env('HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN')
+
+        # Model Load Balancing Configurations.
+        self.MODEL_LB_ENABLED = get_bool_env('MODEL_LB_ENABLED')
+
+        # Platform Billing Configurations.
+        self.BILLING_ENABLED = get_bool_env('BILLING_ENABLED')
+
+        # ------------------------
+        # Enterprise feature Configurations.
+        # **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
+        # ------------------------
+        self.ENTERPRISE_ENABLED = get_bool_env('ENTERPRISE_ENABLED')
+        self.CAN_REPLACE_LOGO = get_bool_env('CAN_REPLACE_LOGO')

api/configs/app_config.py

@@ -1,45 +0,0 @@
-from pydantic_settings import BaseSettings, SettingsConfigDict
-
-from configs.deploy import DeploymentConfig
-from configs.enterprise import EnterpriseFeatureConfig
-from configs.extra import ExtraServiceConfig
-from configs.feature import FeatureConfig
-from configs.middleware import MiddlewareConfig
-from configs.packaging import PackagingInfo
-
-
-# TODO: Both `BaseModel` and `BaseSettings` has `model_config` attribute but they are in different types.
-# This inheritance is depends on the order of the classes.
-# It is better to use `BaseSettings` as the base class.
-class DifyConfig(
-    # based on pydantic-settings
-    BaseSettings,
-
-    # Packaging info
-    PackagingInfo,
-
-    # Deployment configs
-    DeploymentConfig,
-
-    # Feature configs
-    FeatureConfig,
-
-    # Middleware configs
-    MiddlewareConfig,
-
-    # Extra service configs
-    ExtraServiceConfig,
-
-    # Enterprise feature configs
-    # **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
-    EnterpriseFeatureConfig,
-):
-
-    model_config = SettingsConfigDict(
-        # read from dotenv format config file
-        env_file='.env',
-        env_file_encoding='utf-8',
-
-        # ignore extra attributes
-        extra='ignore',
-    )

api/configs/deploy/__init__.py

@@ -1,26 +0,0 @@
-from pydantic import BaseModel, Field
-
-
-class DeploymentConfig(BaseModel):
-    """
-    Deployment configs
-    """
-    APPLICATION_NAME: str = Field(
-        description='application name',
-        default='langgenius/dify',
-    )
-
-    TESTING: bool = Field(
-        description='',
-        default=False,
-    )
-
-    EDITION: str = Field(
-        description='deployment edition',
-        default='SELF_HOSTED',
-    )
-
-    DEPLOY_ENV: str = Field(
-        description='deployment environment, default to PRODUCTION.',
-        default='PRODUCTION',
-    )

api/configs/enterprise/__init__.py

@@ -1,18 +0,0 @@
-from pydantic import BaseModel, Field
-
-
-class EnterpriseFeatureConfig(BaseModel):
-    """
-    Enterprise feature configs.
-    **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
-    """
-    ENTERPRISE_ENABLED: bool = Field(
-        description='whether to enable enterprise features.'
-                    'Before using, please contact business@dify.ai by email to inquire about licensing matters.',
-        default=False,
-    )
-
-    CAN_REPLACE_LOGO: bool = Field(
-        description='whether to allow replacing enterprise logo.',
-        default=False,
-    )

api/configs/extra/__init__.py

@@ -1,12 +0,0 @@
-from pydantic import BaseModel
-
-from configs.extra.notion_config import NotionConfig
-from configs.extra.sentry_config import SentryConfig
-
-
-class ExtraServiceConfig(
-    # place the configs in alphabet order
-    NotionConfig,
-    SentryConfig,
-):
-    pass

api/configs/extra/notion_config.py

@@ -1,33 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field
-
-
-class NotionConfig(BaseModel):
-    """
-    Notion integration configs
-    """
-    NOTION_CLIENT_ID: Optional[str] = Field(
-        description='Notion client ID',
-        default=None,
-    )
-
-    NOTION_CLIENT_SECRET: Optional[str] = Field(
-        description='Notion client secret key',
-        default=None,
-    )
-
-    NOTION_INTEGRATION_TYPE: Optional[str] = Field(
-        description='Notion integration type, default to None, available values: internal.',
-        default=None,
-    )
-
-    NOTION_INTERNAL_SECRET: Optional[str] = Field(
-        description='Notion internal secret key',
-        default=None,
-    )
-
-    NOTION_INTEGRATION_TOKEN: Optional[str] = Field(
-        description='Notion integration token',
-        default=None,
-    )

api/configs/extra/sentry_config.py

@@ -1,23 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field, NonNegativeFloat
-
-
-class SentryConfig(BaseModel):
-    """
-    Sentry configs
-    """
-    SENTRY_DSN: Optional[str] = Field(
-        description='Sentry DSN',
-        default=None,
-    )
-
-    SENTRY_TRACES_SAMPLE_RATE: NonNegativeFloat = Field(
-        description='Sentry trace sample rate',
-        default=1.0,
-    )
-
-    SENTRY_PROFILES_SAMPLE_RATE: NonNegativeFloat = Field(
-        description='Sentry profiles sample rate',
-        default=1.0,
-    )

api/configs/feature/__init__.py

@@ -1,452 +0,0 @@
-from typing import Optional
-
-from pydantic import AliasChoices, BaseModel, Field, NonNegativeInt, PositiveInt, computed_field
-
-from configs.feature.hosted_service import HostedServiceConfig
-
-
-class SecurityConfig(BaseModel):
-    """
-    Secret Key configs
-    """
-    SECRET_KEY: Optional[str] = Field(
-        description='Your App secret key will be used for securely signing the session cookie'
-                    'Make sure you are changing this key for your deployment with a strong key.'
-                    'You can generate a strong key using `openssl rand -base64 42`.'
-                    'Alternatively you can set it with `SECRET_KEY` environment variable.',
-        default=None,
-    )
-
-
-class AppExecutionConfig(BaseModel):
-    """
-    App Execution configs
-    """
-    APP_MAX_EXECUTION_TIME: PositiveInt = Field(
-        description='execution timeout in seconds for app execution',
-        default=1200,
-    )
-
-
-class CodeExecutionSandboxConfig(BaseModel):
-    """
-    Code Execution Sandbox configs
-    """
-    CODE_EXECUTION_ENDPOINT: str = Field(
-        description='endpoint URL of code execution servcie',
-        default='http://sandbox:8194',
-    )
-
-    CODE_EXECUTION_API_KEY: str = Field(
-        description='API key for code execution service',
-        default='dify-sandbox',
-    )
-
-
-class EndpointConfig(BaseModel):
-    """
-    Module URL configs
-    """
-    CONSOLE_API_URL: str = Field(
-        description='The backend URL prefix of the console API.'
-                    'used to concatenate the login authorization callback or notion integration callback.',
-        default='https://cloud.dify.ai',
-    )
-
-    CONSOLE_WEB_URL: str = Field(
-        description='The front-end URL prefix of the console web.'
-                    'used to concatenate some front-end addresses and for CORS configuration use.',
-        default='https://cloud.dify.ai',
-    )
-
-    SERVICE_API_URL: str = Field(
-        description='Service API Url prefix.'
-                    'used to display Service API Base Url to the front-end.',
-        default='https://api.dify.ai',
-    )
-
-    APP_WEB_URL: str = Field(
-        description='WebApp Url prefix.'
-                    'used to display WebAPP API Base Url to the front-end.',
-        default='https://udify.app',
-    )
-
-
-class FileAccessConfig(BaseModel):
-    """
-    File Access configs
-    """
-    FILES_URL: str = Field(
-        description='File preview or download Url prefix.'
-                    ' used to display File preview or download Url to the front-end or as Multi-model inputs;'
-                    'Url is signed and has expiration time.',
-        validation_alias=AliasChoices('FILES_URL', 'CONSOLE_API_URL'),
-        alias_priority=1,
-        default='https://cloud.dify.ai',
-    )
-
-    FILES_ACCESS_TIMEOUT: int = Field(
-        description='timeout in seconds for file accessing',
-        default=300,
-    )
-
-
-class FileUploadConfig(BaseModel):
-    """
-    File Uploading configs
-    """
-    UPLOAD_FILE_SIZE_LIMIT: NonNegativeInt = Field(
-        description='size limit in Megabytes for uploading files',
-        default=15,
-    )
-
-    UPLOAD_FILE_BATCH_LIMIT: NonNegativeInt = Field(
-        description='batch size limit for uploading files',
-        default=5,
-    )
-
-    UPLOAD_IMAGE_FILE_SIZE_LIMIT: NonNegativeInt = Field(
-        description='image file size limit in Megabytes for uploading files',
-        default=10,
-    )
-
-    BATCH_UPLOAD_LIMIT: NonNegativeInt = Field(
-        description='',  # todo: to be clarified
-        default=20,
-    )
-
-
-class HttpConfig(BaseModel):
-    """
-    HTTP configs
-    """
-    API_COMPRESSION_ENABLED: bool = Field(
-        description='whether to enable HTTP response compression of gzip',
-        default=False,
-    )
-
-    inner_CONSOLE_CORS_ALLOW_ORIGINS: str = Field(
-        description='',
-        validation_alias=AliasChoices('CONSOLE_CORS_ALLOW_ORIGINS', 'CONSOLE_WEB_URL'),
-        default='',
-    )
-
-    @computed_field
-    @property
-    def CONSOLE_CORS_ALLOW_ORIGINS(self) -> list[str]:
-        return self.inner_CONSOLE_CORS_ALLOW_ORIGINS.split(',')
-
-    inner_WEB_API_CORS_ALLOW_ORIGINS: str = Field(
-        description='',
-        validation_alias=AliasChoices('WEB_API_CORS_ALLOW_ORIGINS'),
-        default='*',
-    )
-
-    @computed_field
-    @property
-    def WEB_API_CORS_ALLOW_ORIGINS(self) -> list[str]:
-        return self.inner_WEB_API_CORS_ALLOW_ORIGINS.split(',')
-
-
-class InnerAPIConfig(BaseModel):
-    """
-    Inner API configs
-    """
-    INNER_API: bool = Field(
-        description='whether to enable the inner API',
-        default=False,
-    )
-
-    INNER_API_KEY: Optional[str] = Field(
-        description='The inner API key is used to authenticate the inner API',
-        default=None,
-    )
-
-
-class LoggingConfig(BaseModel):
-    """
-    Logging configs
-    """
-
-    LOG_LEVEL: str = Field(
-        description='Log output level, default to INFO.'
-                    'It is recommended to set it to ERROR for production.',
-        default='INFO',
-    )
-
-    LOG_FILE: Optional[str] = Field(
-        description='logging output file path',
-        default=None,
-    )
-
-    LOG_FORMAT: str = Field(
-        description='log format',
-        default='%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s',
-    )
-
-    LOG_DATEFORMAT: Optional[str] = Field(
-        description='log date format',
-        default=None,
-    )
-
-    LOG_TZ: Optional[str] = Field(
-        description='specify log timezone, eg: America/New_York',
-        default=None,
-    )
-
-
-class ModelLoadBalanceConfig(BaseModel):
-    """
-    Model load balance configs
-    """
-    MODEL_LB_ENABLED: bool = Field(
-        description='whether to enable model load balancing',
-        default=False,
-    )
-
-
-class BillingConfig(BaseModel):
-    """
-    Platform Billing Configurations
-    """
-    BILLING_ENABLED: bool = Field(
-        description='whether to enable billing',
-        default=False,
-    )
-
-
-class UpdateConfig(BaseModel):
-    """
-    Update configs
-    """
-    CHECK_UPDATE_URL: str = Field(
-        description='url for checking updates',
-        default='https://updates.dify.ai',
-    )
-
-
-class WorkflowConfig(BaseModel):
-    """
-    Workflow feature configs
-    """
-
-    WORKFLOW_MAX_EXECUTION_STEPS: PositiveInt = Field(
-        description='max execution steps in single workflow execution',
-        default=500,
-    )
-
-    WORKFLOW_MAX_EXECUTION_TIME: PositiveInt = Field(
-        description='max execution time in seconds in single workflow execution',
-        default=1200,
-    )
-
-    WORKFLOW_CALL_MAX_DEPTH: PositiveInt = Field(
-        description='max depth of calling in single workflow execution',
-        default=5,
-    )
-
-
-class OAuthConfig(BaseModel):
-    """
-    oauth configs
-    """
-    OAUTH_REDIRECT_PATH: str = Field(
-        description='redirect path for OAuth',
-        default='/console/api/oauth/authorize',
-    )
-
-    GITHUB_CLIENT_ID: Optional[str] = Field(
-        description='GitHub client id for OAuth',
-        default=None,
-    )
-
-    GITHUB_CLIENT_SECRET: Optional[str] = Field(
-        description='GitHub client secret key for OAuth',
-        default=None,
-    )
-
-    GOOGLE_CLIENT_ID: Optional[str] = Field(
-        description='Google client id for OAuth',
-        default=None,
-    )
-
-    GOOGLE_CLIENT_SECRET: Optional[str] = Field(
-        description='Google client secret key for OAuth',
-        default=None,
-    )
-
-
-class ModerationConfig(BaseModel):
-    """
-    Moderation in app configs.
-    """
-
-    # todo: to be clarified in usage and unit
-    OUTPUT_MODERATION_BUFFER_SIZE: PositiveInt = Field(
-        description='buffer size for moderation',
-        default=300,
-    )
-
-
-class ToolConfig(BaseModel):
-    """
-    Tool configs
-    """
-
-    TOOL_ICON_CACHE_MAX_AGE: PositiveInt = Field(
-        description='max age in seconds for tool icon caching',
-        default=3600,
-    )
-
-
-class MailConfig(BaseModel):
-    """
-    Mail Configurations
-    """
-
-    MAIL_TYPE: Optional[str] = Field(
-        description='Mail provider type name, default to None, availabile values are `smtp` and `resend`.',
-        default=None,
-    )
-
-    MAIL_DEFAULT_SEND_FROM: Optional[str] = Field(
-        description='default email address for sending from ',
-        default=None,
-    )
-
-    RESEND_API_KEY: Optional[str] = Field(
-        description='API key for Resend',
-        default=None,
-    )
-
-    RESEND_API_URL: Optional[str] = Field(
-        description='API URL for Resend',
-        default=None,
-    )
-
-    SMTP_SERVER: Optional[str] = Field(
-        description='smtp server host',
-        default=None,
-    )
-
-    SMTP_PORT: Optional[int] = Field(
-        description='smtp server port',
-        default=465,
-    )
-
-    SMTP_USERNAME: Optional[str] = Field(
-        description='smtp server username',
-        default=None,
-    )
-
-    SMTP_PASSWORD: Optional[str] = Field(
-        description='smtp server password',
-        default=None,
-    )
-
-    SMTP_USE_TLS: bool = Field(
-        description='whether to use TLS connection to smtp server',
-        default=False,
-    )
-
-    SMTP_OPPORTUNISTIC_TLS: bool = Field(
-        description='whether to use opportunistic TLS connection to smtp server',
-        default=False,
-    )
-
-
-class RagEtlConfig(BaseModel):
-    """
-    RAG ETL Configurations.
-    """
-
-    ETL_TYPE: str = Field(
-        description='RAG ETL type name, default to `dify`, available values are `dify` and `Unstructured`. ',
-        default='dify',
-    )
-
-    KEYWORD_DATA_SOURCE_TYPE: str = Field(
-        description='source type for keyword data, default to `database`, available values are `database` .',
-        default='database',
-    )
-
-    UNSTRUCTURED_API_URL: Optional[str] = Field(
-        description='API URL for Unstructured',
-        default=None,
-    )
-
-    UNSTRUCTURED_API_KEY: Optional[str] = Field(
-        description='API key for Unstructured',
-        default=None,
-    )
-
-
-class DataSetConfig(BaseModel):
-    """
-    Dataset configs
-    """
-
-    CLEAN_DAY_SETTING: PositiveInt = Field(
-        description='interval in days for cleaning up dataset',
-        default=30,
-    )
-
-
-class WorkspaceConfig(BaseModel):
-    """
-    Workspace configs
-    """
-
-    INVITE_EXPIRY_HOURS: PositiveInt = Field(
-        description='workspaces invitation expiration in hours',
-        default=72,
-    )
-
-
-class IndexingConfig(BaseModel):
-    """
-    Indexing configs.
-    """
-
-    INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: PositiveInt = Field(
-        description='max segmentation token length for indexing',
-        default=1000,
-    )
-
-
-class ImageFormatConfig(BaseModel):
-    MULTIMODAL_SEND_IMAGE_FORMAT: str = Field(
-        description='multi model send image format, support base64, url, default is base64',
-        default='base64',
-    )
-
-
-class FeatureConfig(
-    # place the configs in alphabet order
-    AppExecutionConfig,
-    BillingConfig,
-    CodeExecutionSandboxConfig,
-    DataSetConfig,
-    EndpointConfig,
-    FileAccessConfig,
-    FileUploadConfig,
-    HttpConfig,
-    ImageFormatConfig,
-    InnerAPIConfig,
-    IndexingConfig,
-    LoggingConfig,
-    MailConfig,
-    ModelLoadBalanceConfig,
-    ModerationConfig,
-    OAuthConfig,
-    RagEtlConfig,
-    SecurityConfig,
-    ToolConfig,
-    UpdateConfig,
-    WorkflowConfig,
-    WorkspaceConfig,
-
-    # hosted services config
-    HostedServiceConfig,
-):
-    pass

api/configs/feature/hosted_service/__init__.py

@@ -1,208 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field, NonNegativeInt
-
-
-class HostedOpenAiConfig(BaseModel):
-    """
-    Hosted OpenAI service config
-    """
-
-    HOSTED_OPENAI_API_KEY: Optional[str] = Field(
-        description='',
-        default=None,
-    )
-
-    HOSTED_OPENAI_API_BASE: Optional[str] = Field(
-        description='',
-        default=None,
-    )
-
-    HOSTED_OPENAI_API_ORGANIZATION: Optional[str] = Field(
-        description='',
-        default=None,
-    )
-
-    HOSTED_OPENAI_TRIAL_ENABLED: bool = Field(
-        description='',
-        default=False,
-    )
-
-    HOSTED_OPENAI_TRIAL_MODELS: str = Field(
-        description='',
-        default='gpt-3.5-turbo,'
-                'gpt-3.5-turbo-1106,'
-                'gpt-3.5-turbo-instruct,'
-                'gpt-3.5-turbo-16k,'
-                'gpt-3.5-turbo-16k-0613,'
-                'gpt-3.5-turbo-0613,'
-                'gpt-3.5-turbo-0125,'
-                'text-davinci-003',
-    )
-
-    HOSTED_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field(
-        description='',
-        default=200,
-    )
-
-    HOSTED_OPENAI_PAID_ENABLED: bool = Field(
-        description='',
-        default=False,
-    )
-
-    HOSTED_OPENAI_PAID_MODELS: str = Field(
-        description='',
-        default='gpt-4,'
-                'gpt-4-turbo-preview,'
-                'gpt-4-turbo-2024-04-09,'
-                'gpt-4-1106-preview,'
-                'gpt-4-0125-preview,'
-                'gpt-3.5-turbo,'
-                'gpt-3.5-turbo-16k,'
-                'gpt-3.5-turbo-16k-0613,'
-                'gpt-3.5-turbo-1106,'
-                'gpt-3.5-turbo-0613,'
-                'gpt-3.5-turbo-0125,'
-                'gpt-3.5-turbo-instruct,'
-                'text-davinci-003',
-    )
-
-
-class HostedAzureOpenAiConfig(BaseModel):
-    """
-    Hosted OpenAI service config
-    """
-
-    HOSTED_AZURE_OPENAI_ENABLED: bool = Field(
-        description='',
-        default=False,
-    )
-
-    HOSTED_OPENAI_API_KEY: Optional[str] = Field(
-        description='',
-        default=None,
-    )
-
-    HOSTED_AZURE_OPENAI_API_BASE: Optional[str] = Field(
-        description='',
-        default=None,
-    )
-
-    HOSTED_AZURE_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field(
-        description='',
-        default=200,
-    )
-
-
-class HostedAnthropicConfig(BaseModel):
-    """
-    Hosted Azure OpenAI service config
-    """
-
-    HOSTED_ANTHROPIC_API_BASE: Optional[str] = Field(
-        description='',
-        default=None,
-    )
-
-    HOSTED_ANTHROPIC_API_KEY: Optional[str] = Field(
-        description='',
-        default=None,
-    )
-
-    HOSTED_ANTHROPIC_TRIAL_ENABLED: bool = Field(
-        description='',
-        default=False,
-    )
-
-    HOSTED_ANTHROPIC_QUOTA_LIMIT: NonNegativeInt = Field(
-        description='',
-        default=600000,
-    )
-
-    HOSTED_ANTHROPIC_PAID_ENABLED: bool = Field(
-        description='',
-        default=False,
-    )
-
-
-class HostedMinmaxConfig(BaseModel):
-    """
-    Hosted Minmax service config
-    """
-
-    HOSTED_MINIMAX_ENABLED: bool = Field(
-        description='',
-        default=False,
-    )
-
-
-class HostedSparkConfig(BaseModel):
-    """
-    Hosted Spark service config
-    """
-
-    HOSTED_SPARK_ENABLED: bool = Field(
-        description='',
-        default=False,
-    )
-
-
-class HostedZhipuAIConfig(BaseModel):
-    """
-    Hosted Minmax service config
-    """
-
-    HOSTED_ZHIPUAI_ENABLED: bool = Field(
-        description='',
-        default=False,
-    )
-
-
-class HostedModerationConfig(BaseModel):
-    """
-    Hosted Moderation service config
-    """
-
-    HOSTED_MODERATION_ENABLED: bool = Field(
-        description='',
-        default=False,
-    )
-
-    HOSTED_MODERATION_PROVIDERS: str = Field(
-        description='',
-        default='',
-    )
-
-
-class HostedFetchAppTemplateConfig(BaseModel):
-    """
-    Hosted Moderation service config
-    """
-
-    HOSTED_FETCH_APP_TEMPLATES_MODE: str = Field(
-        description='the mode for fetching app templates,'
-                    ' default to remote,'
-                    ' available values: remote, db, builtin',
-        default='remote',
-    )
-
-    HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN: str = Field(
-        description='the domain for fetching remote app templates',
-        default='https://tmpl.dify.ai',
-    )
-
-
-class HostedServiceConfig(
-    # place the configs in alphabet order
-    HostedAnthropicConfig,
-    HostedAzureOpenAiConfig,
-    HostedFetchAppTemplateConfig,
-    HostedMinmaxConfig,
-    HostedOpenAiConfig,
-    HostedSparkConfig,
-    HostedZhipuAIConfig,
-
-    # moderation
-    HostedModerationConfig,
-):
-    pass

api/configs/middleware/__init__.py

@@ -1,187 +0,0 @@
-from typing import Any, Optional
-
-from pydantic import BaseModel, Field, NonNegativeInt, PositiveInt, computed_field
-
-from configs.middleware.cache.redis_config import RedisConfig
-from configs.middleware.storage.aliyun_oss_storage_config import AliyunOSSStorageConfig
-from configs.middleware.storage.amazon_s3_storage_config import S3StorageConfig
-from configs.middleware.storage.azure_blob_storage_config import AzureBlobStorageConfig
-from configs.middleware.storage.google_cloud_storage_config import GoogleCloudStorageConfig
-from configs.middleware.storage.oci_storage_config import OCIStorageConfig
-from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
-from configs.middleware.vdb.chroma_config import ChromaConfig
-from configs.middleware.vdb.milvus_config import MilvusConfig
-from configs.middleware.vdb.opensearch_config import OpenSearchConfig
-from configs.middleware.vdb.oracle_config import OracleConfig
-from configs.middleware.vdb.pgvector_config import PGVectorConfig
-from configs.middleware.vdb.pgvectors_config import PGVectoRSConfig
-from configs.middleware.vdb.qdrant_config import QdrantConfig
-from configs.middleware.vdb.relyt_config import RelytConfig
-from configs.middleware.vdb.tencent_vector_config import TencentVectorDBConfig
-from configs.middleware.vdb.tidb_vector_config import TiDBVectorConfig
-from configs.middleware.vdb.weaviate_config import WeaviateConfig
-
-
-class StorageConfig(BaseModel):
-    STORAGE_TYPE: str = Field(
-        description='storage type,'
-                    ' default to `local`,'
-                    ' available values are `local`, `s3`, `azure-blob`, `aliyun-oss`, `google-storage`.',
-        default='local',
-    )
-
-    STORAGE_LOCAL_PATH: str = Field(
-        description='local storage path',
-        default='storage',
-    )
-
-
-class VectorStoreConfig(BaseModel):
-    VECTOR_STORE: Optional[str] = Field(
-        description='vector store type',
-        default=None,
-    )
-
-
-class KeywordStoreConfig(BaseModel):
-    KEYWORD_STORE: str = Field(
-        description='keyword store type',
-        default='jieba',
-    )
-
-
-class DatabaseConfig:
-    DB_HOST: str = Field(
-        description='db host',
-        default='localhost',
-    )
-
-    DB_PORT: PositiveInt = Field(
-        description='db port',
-        default=5432,
-    )
-
-    DB_USERNAME: str = Field(
-        description='db username',
-        default='postgres',
-    )
-
-    DB_PASSWORD: str = Field(
-        description='db password',
-        default='',
-    )
-
-    DB_DATABASE: str = Field(
-        description='db database',
-        default='dify',
-    )
-
-    DB_CHARSET: str = Field(
-        description='db charset',
-        default='',
-    )
-
-    SQLALCHEMY_DATABASE_URI_SCHEME: str = Field(
-        description='db uri scheme',
-        default='postgresql',
-    )
-
-    @computed_field
-    @property
-    def SQLALCHEMY_DATABASE_URI(self) -> str:
-        db_extras = f"?client_encoding={self.DB_CHARSET}" if self.DB_CHARSET else ""
-        return (f"{self.SQLALCHEMY_DATABASE_URI_SCHEME}://"
-                f"{self.DB_USERNAME}:{self.DB_PASSWORD}@{self.DB_HOST}:{self.DB_PORT}/{self.DB_DATABASE}"
-                f"{db_extras}")
-
-    SQLALCHEMY_POOL_SIZE: NonNegativeInt = Field(
-        description='pool size of SqlAlchemy',
-        default=30,
-    )
-
-    SQLALCHEMY_MAX_OVERFLOW: NonNegativeInt = Field(
-        description='max overflows for SqlAlchemy',
-        default=10,
-    )
-
-    SQLALCHEMY_POOL_RECYCLE: NonNegativeInt = Field(
-        description='SqlAlchemy pool recycle',
-        default=3600,
-    )
-
-    SQLALCHEMY_POOL_PRE_PING: bool = Field(
-        description='whether to enable pool pre-ping in SqlAlchemy',
-        default=False,
-    )
-
-    SQLALCHEMY_ECHO: bool = Field(
-        description='whether to enable SqlAlchemy echo',
-        default=False,
-    )
-
-    @computed_field
-    @property
-    def SQLALCHEMY_ENGINE_OPTIONS(self) -> dict[str, Any]:
-        return {
-            'pool_size': self.SQLALCHEMY_POOL_SIZE,
-            'max_overflow': self.SQLALCHEMY_MAX_OVERFLOW,
-            'pool_recycle': self.SQLALCHEMY_POOL_RECYCLE,
-            'pool_pre_ping': self.SQLALCHEMY_POOL_PRE_PING,
-            'connect_args': {'options': '-c timezone=UTC'},
-        }
-
-
-class CeleryConfig(DatabaseConfig):
-    CELERY_BACKEND: str = Field(
-        description='Celery backend, available values are `database`, `redis`',
-        default='database',
-    )
-
-    CELERY_BROKER_URL: Optional[str] = Field(
-        description='CELERY_BROKER_URL',
-        default=None,
-    )
-
-    @computed_field
-    @property
-    def CELERY_RESULT_BACKEND(self) -> str | None:
-        return 'db+{}'.format(self.SQLALCHEMY_DATABASE_URI) \
-            if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL
-
-    @computed_field
-    @property
-    def BROKER_USE_SSL(self) -> bool:
-        return self.CELERY_BROKER_URL.startswith('rediss://') if self.CELERY_BROKER_URL else False
-
-
-class MiddlewareConfig(
-    # place the configs in alphabet order
-    CeleryConfig,
-    DatabaseConfig,
-    KeywordStoreConfig,
-    RedisConfig,
-
-    # configs of storage and storage providers
-    StorageConfig,
-    AliyunOSSStorageConfig,
-    AzureBlobStorageConfig,
-    GoogleCloudStorageConfig,
-    TencentCloudCOSStorageConfig,
-    S3StorageConfig,
-    OCIStorageConfig,
-
-    # configs of vdb and vdb providers
-    VectorStoreConfig,
-    ChromaConfig,
-    MilvusConfig,
-    OpenSearchConfig,
-    OracleConfig,
-    PGVectorConfig,
-    PGVectoRSConfig,
-    QdrantConfig,
-    RelytConfig,
-    TencentVectorDBConfig,
-    TiDBVectorConfig,
-    WeaviateConfig,
-):
-    pass

api/configs/middleware/cache/redis_config.py

@@ -1,38 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field, NonNegativeInt, PositiveInt
-
-
-class RedisConfig(BaseModel):
-    """
-    Redis configs
-    """
-    REDIS_HOST: str = Field(
-        description='Redis host',
-        default='localhost',
-    )
-
-    REDIS_PORT: PositiveInt = Field(
-        description='Redis port',
-        default=6379,
-    )
-
-    REDIS_USERNAME: Optional[str] = Field(
-        description='Redis username',
-        default=None,
-    )
-
-    REDIS_PASSWORD: Optional[str] = Field(
-        description='Redis password',
-        default=None,
-    )
-
-    REDIS_DB: NonNegativeInt = Field(
-        description='Redis database id, default to 0',
-        default=0,
-    )
-
-    REDIS_USE_SSL: bool = Field(
-        description='whether to use SSL for Redis connection',
-        default=False,
-    )

api/configs/middleware/storage/aliyun_oss_storage_config.py

@@ -1,39 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field
-
-
-class AliyunOSSStorageConfig(BaseModel):
-    """
-    Aliyun storage configs
-    """
-
-    ALIYUN_OSS_BUCKET_NAME: Optional[str] = Field(
-        description='Aliyun OSS bucket name',
-        default=None,
-    )
-
-    ALIYUN_OSS_ACCESS_KEY: Optional[str] = Field(
-        description='Aliyun OSS access key',
-        default=None,
-    )
-
-    ALIYUN_OSS_SECRET_KEY: Optional[str] = Field(
-        description='Aliyun OSS secret key',
-        default=None,
-    )
-
-    ALIYUN_OSS_ENDPOINT: Optional[str] = Field(
-        description='Aliyun OSS endpoint URL',
-        default=None,
-    )
-
-    ALIYUN_OSS_REGION: Optional[str] = Field(
-        description='Aliyun OSS region',
-        default=None,
-    )
-
-    ALIYUN_OSS_AUTH_VERSION: Optional[str] = Field(
-        description='Aliyun OSS authentication version',
-        default=None,
-    )

api/configs/middleware/storage/amazon_s3_storage_config.py

@@ -1,44 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field
-
-
-class S3StorageConfig(BaseModel):
-    """
-    S3 storage configs
-    """
-
-    S3_ENDPOINT: Optional[str] = Field(
-        description='S3 storage endpoint',
-        default=None,
-    )
-
-    S3_REGION: Optional[str] = Field(
-        description='S3 storage region',
-        default=None,
-    )
-
-    S3_BUCKET_NAME: Optional[str] = Field(
-        description='S3 storage bucket name',
-        default=None,
-    )
-
-    S3_ACCESS_KEY: Optional[str] = Field(
-        description='S3 storage access key',
-        default=None,
-    )
-
-    S3_SECRET_KEY: Optional[str] = Field(
-        description='S3 storage secret key',
-        default=None,
-    )
-
-    S3_ADDRESS_STYLE: str = Field(
-        description='S3 storage address style',
-        default='auto',
-    )
-
-    S3_USE_AWS_MANAGED_IAM: bool = Field(
-        description='whether to use aws managed IAM for S3',
-        default=False,
-    )

api/configs/middleware/storage/azure_blob_storage_config.py

@@ -1,29 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field
-
-
-class AzureBlobStorageConfig(BaseModel):
-    """
-    Azure Blob storage configs
-    """
-
-    AZURE_BLOB_ACCOUNT_NAME: Optional[str] = Field(
-        description='Azure Blob account name',
-        default=None,
-    )
-
-    AZURE_BLOB_ACCOUNT_KEY: Optional[str] = Field(
-        description='Azure Blob account key',
-        default=None,
-    )
-
-    AZURE_BLOB_CONTAINER_NAME: Optional[str] = Field(
-        description='Azure Blob container name',
-        default=None,
-    )
-
-    AZURE_BLOB_ACCOUNT_URL: Optional[str] = Field(
-        description='Azure Blob account URL',
-        default=None,
-    )

api/configs/middleware/storage/google_cloud_storage_config.py

@@ -1,19 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field
-
-
-class GoogleCloudStorageConfig(BaseModel):
-    """
-    Google Cloud storage configs
-    """
-
-    GOOGLE_STORAGE_BUCKET_NAME: Optional[str] = Field(
-        description='Google Cloud storage bucket name',
-        default=None,
-    )
-
-    GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: Optional[str] = Field(
-        description='Google Cloud storage service account json base64',
-        default=None,
-    )

api/configs/middleware/storage/oci_storage_config.py

@@ -1,35 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field
-
-
-class OCIStorageConfig(BaseModel):
-    """
-    OCI storage configs
-    """
-
-    OCI_ENDPOINT: Optional[str] = Field(
-        description='OCI storage endpoint',
-        default=None,
-    )
-
-    OCI_REGION: Optional[str] = Field(
-        description='OCI storage region',
-        default=None,
-    )
-
-    OCI_BUCKET_NAME: Optional[str] = Field(
-        description='OCI storage bucket name',
-        default=None,
-    )
-
-    OCI_ACCESS_KEY: Optional[str] = Field(
-        description='OCI storage access key',
-        default=None,
-    )
-
-    OCI_SECRET_KEY: Optional[str] = Field(
-        description='OCI storage secret key',
-        default=None,
-    )
-

api/configs/middleware/storage/tencent_cos_storage_config.py

@@ -1,34 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field
-
-
-class TencentCloudCOSStorageConfig(BaseModel):
-    """
-    Tencent Cloud COS storage configs
-    """
-
-    TENCENT_COS_BUCKET_NAME: Optional[str] = Field(
-        description='Tencent Cloud COS bucket name',
-        default=None,
-    )
-
-    TENCENT_COS_REGION: Optional[str] = Field(
-        description='Tencent Cloud COS region',
-        default=None,
-    )
-
-    TENCENT_COS_SECRET_ID: Optional[str] = Field(
-        description='Tencent Cloud COS secret id',
-        default=None,
-    )
-
-    TENCENT_COS_SECRET_KEY: Optional[str] = Field(
-        description='Tencent Cloud COS secret key',
-        default=None,
-    )
-
-    TENCENT_COS_SCHEME: Optional[str] = Field(
-        description='Tencent Cloud COS scheme',
-        default=None,
-    )

api/configs/middleware/vdb/chroma_config.py

@@ -1,39 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field, PositiveInt
-
-
-class ChromaConfig(BaseModel):
-    """
-    Chroma configs
-    """
-
-    CHROMA_HOST: Optional[str] = Field(
-        description='Chroma host',
-        default=None,
-    )
-
-    CHROMA_PORT: PositiveInt = Field(
-        description='Chroma port',
-        default=8000,
-    )
-
-    CHROMA_TENANT: Optional[str] = Field(
-        description='Chroma database',
-        default=None,
-    )
-
-    CHROMA_DATABASE: Optional[str] = Field(
-        description='Chroma database',
-        default=None,
-    )
-
-    CHROMA_AUTH_PROVIDER: Optional[str] = Field(
-        description='Chroma authentication provider',
-        default=None,
-    )
-
-    CHROMA_AUTH_CREDENTIALS: Optional[str] = Field(
-        description='Chroma authentication credentials',
-        default=None,
-    )

api/configs/middleware/vdb/milvus_config.py

@@ -1,39 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field, PositiveInt
-
-
-class MilvusConfig(BaseModel):
-    """
-    Milvus configs
-    """
-
-    MILVUS_HOST: Optional[str] = Field(
-        description='Milvus host',
-        default=None,
-    )
-
-    MILVUS_PORT: PositiveInt = Field(
-        description='Milvus RestFul API port',
-        default=9091,
-    )
-
-    MILVUS_USER: Optional[str] = Field(
-        description='Milvus user',
-        default=None,
-    )
-
-    MILVUS_PASSWORD: Optional[str] = Field(
-        description='Milvus password',
-        default=None,
-    )
-
-    MILVUS_SECURE: bool = Field(
-        description='whether to use SSL connection for Milvus',
-        default=False,
-    )
-
-    MILVUS_DATABASE: str = Field(
-        description='Milvus database, default to `default`',
-        default='default',
-    )

api/configs/middleware/vdb/opensearch_config.py

@@ -1,34 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field, PositiveInt
-
-
-class OpenSearchConfig(BaseModel):
-    """
-    OpenSearch configs
-    """
-
-    OPENSEARCH_HOST: Optional[str] = Field(
-        description='OpenSearch host',
-        default=None,
-    )
-
-    OPENSEARCH_PORT: PositiveInt = Field(
-        description='OpenSearch port',
-        default=9200,
-    )
-
-    OPENSEARCH_USER: Optional[str] = Field(
-        description='OpenSearch user',
-        default=None,
-    )
-
-    OPENSEARCH_PASSWORD: Optional[str] = Field(
-        description='OpenSearch password',
-        default=None,
-    )
-
-    OPENSEARCH_SECURE: bool = Field(
-        description='whether to use SSL connection for OpenSearch',
-        default=False,
-    )

api/configs/middleware/vdb/oracle_config.py

@@ -1,34 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field, PositiveInt
-
-
-class OracleConfig(BaseModel):
-    """
-    ORACLE configs
-    """
-
-    ORACLE_HOST: Optional[str] = Field(
-        description='ORACLE host',
-        default=None,
-    )
-
-    ORACLE_PORT: Optional[PositiveInt] = Field(
-        description='ORACLE port',
-        default=1521,
-    )
-
-    ORACLE_USER: Optional[str] = Field(
-        description='ORACLE user',
-        default=None,
-    )
-
-    ORACLE_PASSWORD: Optional[str] = Field(
-        description='ORACLE password',
-        default=None,
-    )
-
-    ORACLE_DATABASE: Optional[str] = Field(
-        description='ORACLE database',
-        default=None,
-    )

api/configs/middleware/vdb/pgvector_config.py

@@ -1,34 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field, PositiveInt
-
-
-class PGVectorConfig(BaseModel):
-    """
-    PGVector configs
-    """
-
-    PGVECTOR_HOST: Optional[str] = Field(
-        description='PGVector host',
-        default=None,
-    )
-
-    PGVECTOR_PORT: Optional[PositiveInt] = Field(
-        description='PGVector port',
-        default=5433,
-    )
-
-    PGVECTOR_USER: Optional[str] = Field(
-        description='PGVector user',
-        default=None,
-    )
-
-    PGVECTOR_PASSWORD: Optional[str] = Field(
-        description='PGVector password',
-        default=None,
-    )
-
-    PGVECTOR_DATABASE: Optional[str] = Field(
-        description='PGVector database',
-        default=None,
-    )

api/configs/middleware/vdb/pgvectors_config.py

@@ -1,34 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field, PositiveInt
-
-
-class PGVectoRSConfig(BaseModel):
-    """
-    PGVectoRS configs
-    """
-
-    PGVECTO_RS_HOST: Optional[str] = Field(
-        description='PGVectoRS host',
-        default=None,
-    )
-
-    PGVECTO_RS_PORT: Optional[PositiveInt] = Field(
-        description='PGVectoRS port',
-        default=5431,
-    )
-
-    PGVECTO_RS_USER: Optional[str] = Field(
-        description='PGVectoRS user',
-        default=None,
-    )
-
-    PGVECTO_RS_PASSWORD: Optional[str] = Field(
-        description='PGVectoRS password',
-        default=None,
-    )
-
-    PGVECTO_RS_DATABASE: Optional[str] = Field(
-        description='PGVectoRS database',
-        default=None,
-    )

api/configs/middleware/vdb/qdrant_config.py

@@ -1,34 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field, NonNegativeInt, PositiveInt
-
-
-class QdrantConfig(BaseModel):
-    """
-    Qdrant configs
-    """
-
-    QDRANT_URL: Optional[str] = Field(
-        description='Qdrant url',
-        default=None,
-    )
-
-    QDRANT_API_KEY: Optional[str] = Field(
-        description='Qdrant api key',
-        default=None,
-    )
-
-    QDRANT_CLIENT_TIMEOUT: NonNegativeInt = Field(
-        description='Qdrant client timeout in seconds',
-        default=20,
-    )
-
-    QDRANT_GRPC_ENABLED: bool = Field(
-        description='whether enable grpc support for Qdrant connection',
-        default=False,
-    )
-
-    QDRANT_GRPC_PORT: PositiveInt = Field(
-        description='Qdrant grpc port',
-        default=6334,
-    )

api/configs/middleware/vdb/relyt_config.py

@@ -1,34 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field, PositiveInt
-
-
-class RelytConfig(BaseModel):
-    """
-    Relyt configs
-    """
-
-    RELYT_HOST: Optional[str] = Field(
-        description='Relyt host',
-        default=None,
-    )
-
-    RELYT_PORT: PositiveInt = Field(
-        description='Relyt port',
-        default=9200,
-    )
-
-    RELYT_USER: Optional[str] = Field(
-        description='Relyt user',
-        default=None,
-    )
-
-    RELYT_PASSWORD: Optional[str] = Field(
-        description='Relyt password',
-        default=None,
-    )
-
-    RELYT_DATABASE: Optional[str] = Field(
-        description='Relyt database',
-        default='default',
-    )

api/configs/middleware/vdb/tencent_vector_config.py

@@ -1,44 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field, PositiveInt
-
-
-class TencentVectorDBConfig(BaseModel):
-    """
-    Tencent Vector configs
-    """
-
-    TENCENT_VECTOR_DB_URL: Optional[str] = Field(
-        description='Tencent Vector URL',
-        default=None,
-    )
-
-    TENCENT_VECTOR_DB_API_KEY: Optional[str] = Field(
-        description='Tencent Vector API key',
-        default=None,
-    )
-
-    TENCENT_VECTOR_DB_TIMEOUT: PositiveInt = Field(
-        description='Tencent Vector timeout in seconds',
-        default=30,
-    )
-
-    TENCENT_VECTOR_DB_USERNAME: Optional[str] = Field(
-        description='Tencent Vector password',
-        default=None,
-    )
-
-    TENCENT_VECTOR_DB_PASSWORD: Optional[str] = Field(
-        description='Tencent Vector password',
-        default=None,
-    )
-
-    TENCENT_VECTOR_DB_SHARD: PositiveInt = Field(
-        description='Tencent Vector sharding number',
-        default=1,
-    )
-
-    TENCENT_VECTOR_DB_REPLICAS: PositiveInt = Field(
-        description='Tencent Vector replicas',
-        default=2,
-    )

api/configs/middleware/vdb/tidb_vector_config.py

@@ -1,34 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field, PositiveInt
-
-
-class TiDBVectorConfig(BaseModel):
-    """
-    TiDB Vector configs
-    """
-
-    TIDB_VECTOR_HOST: Optional[str] = Field(
-        description='TiDB Vector host',
-        default=None,
-    )
-
-    TIDB_VECTOR_PORT: Optional[PositiveInt] = Field(
-        description='TiDB Vector port',
-        default=4000,
-    )
-
-    TIDB_VECTOR_USER: Optional[str] = Field(
-        description='TiDB Vector user',
-        default=None,
-    )
-
-    TIDB_VECTOR_PASSWORD: Optional[str] = Field(
-        description='TiDB Vector password',
-        default=None,
-    )
-
-    TIDB_VECTOR_DATABASE: Optional[str] = Field(
-        description='TiDB Vector database',
-        default=None,
-    )

api/configs/middleware/vdb/weaviate_config.py

@@ -1,29 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field, PositiveInt
-
-
-class WeaviateConfig(BaseModel):
-    """
-    Weaviate configs
-    """
-
-    WEAVIATE_ENDPOINT: Optional[str] = Field(
-        description='Weaviate endpoint URL',
-        default=None,
-    )
-
-    WEAVIATE_API_KEY: Optional[str] = Field(
-        description='Weaviate API key',
-        default=None,
-    )
-
-    WEAVIATE_GRPC_ENABLED: bool = Field(
-        description='whether to enable gRPC for Weaviate connection',
-        default=True,
-    )
-
-    WEAVIATE_BATCH_SIZE: PositiveInt = Field(
-        description='Weaviate batch size',
-        default=100,
-    )

api/configs/packaging/__init__.py

@@ -1,17 +0,0 @@
-from pydantic import BaseModel, Field
-
-
-class PackagingInfo(BaseModel):
-    """
-    Packaging build information
-    """
-
-    CURRENT_VERSION: str = Field(
-        description='Dify version',
-        default='0.6.12-fix1',
-    )
-
-    COMMIT_SHA: str = Field(
-        description="SHA-1 checksum of the git commit used to build the app",
-        default='',
-    )

api/constants/languages.py

@@ -1,3 +1,7 @@
+
+
+languages = ['en-US', 'zh-Hans', 'zh-Hant', 'pt-BR', 'es-ES', 'fr-FR', 'de-DE', 'ja-JP', 'ko-KR', 'ru-RU', 'it-IT', 'uk-UA', 'vi-VN', 'pl-PL', 'hi-IN']
+
 language_timezone_mapping = {
     'en-US': 'America/New_York',
     'zh-Hans': 'Asia/Shanghai',
@@ -17,8 +21,6 @@ language_timezone_mapping = {
     'hi-IN': 'Asia/Kolkata'
 }
 
-languages = list(language_timezone_mapping.keys())
-
 
 def supported_language(lang):
     if lang in languages:

api/constants/model_template.py

@@ -22,7 +22,7 @@ default_app_templates = {
         'model_config': {
             'model': {
                 "provider": "openai",
-                "name": "gpt-4o",
+                "name": "gpt-4",
                 "mode": "chat",
                 "completion_params": {}
             },
@@ -51,7 +51,7 @@ default_app_templates = {
         'model_config': {
             'model': {
                 "provider": "openai",
-                "name": "gpt-4o",
+                "name": "gpt-4",
                 "mode": "chat",
                 "completion_params": {}
             }
@@ -77,7 +77,7 @@ default_app_templates = {
         'model_config': {
             'model': {
                 "provider": "openai",
-                "name": "gpt-4o",
+                "name": "gpt-4",
                 "mode": "chat",
                 "completion_params": {}
             }

api/controllers/console/__init__.py

@@ -20,7 +20,6 @@ from .app import (
     generator,
     message,
     model_config,
-    ops_trace,
     site,
     statistic,
     workflow,

api/controllers/console/app/app.py

@@ -1,3 +1,4 @@
+import json
 import uuid
 
 from flask_login import current_user
@@ -8,14 +9,17 @@ from controllers.console import api
 from controllers.console.app.wraps import get_app_model
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
-from core.ops.ops_trace_manager import OpsTraceManager
+from core.tools.tool_manager import ToolManager
+from core.tools.utils.configuration import ToolParameterConfigurationManager
 from fields.app_fields import (
     app_detail_fields,
     app_detail_fields_with_site,
     app_pagination_fields,
 )
 from libs.login import login_required
+from models.model import App, AppMode, AppModelConfig
 from services.app_service import AppService
+from services.tag_service import TagService
 
 ALLOW_CREATE_APP_MODES = ['chat', 'agent-chat', 'advanced-chat', 'workflow', 'completion']
 
@@ -190,10 +194,6 @@ class AppExportApi(Resource):
     @get_app_model
     def get(self, app_model):
         """Export app"""
-        # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
-            raise Forbidden()
-
         app_service = AppService()
 
         return {
@@ -286,39 +286,6 @@ class AppApiStatus(Resource):
         return app_model
 
 
-class AppTraceApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def get(self, app_id):
-        """Get app trace"""
-        app_trace_config = OpsTraceManager.get_app_tracing_config(
-            app_id=app_id
-        )
-
-        return app_trace_config
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def post(self, app_id):
-        # add app trace
-        if not current_user.is_admin_or_owner:
-            raise Forbidden()
-        parser = reqparse.RequestParser()
-        parser.add_argument('enabled', type=bool, required=True, location='json')
-        parser.add_argument('tracing_provider', type=str, required=True, location='json')
-        args = parser.parse_args()
-
-        OpsTraceManager.update_app_tracing_config(
-            app_id=app_id,
-            enabled=args['enabled'],
-            tracing_provider=args['tracing_provider'],
-        )
-
-        return {"result": "success"}
-
-
 api.add_resource(AppListApi, '/apps')
 api.add_resource(AppImportApi, '/apps/import')
 api.add_resource(AppApi, '/apps/<uuid:app_id>')
@@ -328,4 +295,3 @@ api.add_resource(AppNameApi, '/apps/<uuid:app_id>/name')
 api.add_resource(AppIconApi, '/apps/<uuid:app_id>/icon')
 api.add_resource(AppSiteStatus, '/apps/<uuid:app_id>/site-enable')
 api.add_resource(AppApiStatus, '/apps/<uuid:app_id>/api-enable')
-api.add_resource(AppTraceApi, '/apps/<uuid:app_id>/trace')

api/controllers/console/app/error.py

@@ -97,21 +97,3 @@ class DraftWorkflowNotSync(BaseHTTPException):
     error_code = 'draft_workflow_not_sync'
     description = "Workflow graph might have been modified, please refresh and resubmit."
     code = 400
-
-
-class TracingConfigNotExist(BaseHTTPException):
-    error_code = 'trace_config_not_exist'
-    description = "Trace config not exist."
-    code = 400
-
-
-class TracingConfigIsExist(BaseHTTPException):
-    error_code = 'trace_config_is_exist'
-    description = "Trace config is exist."
-    code = 400
-
-
-class TracingConfigCheckError(BaseHTTPException):
-    error_code = 'trace_config_check_error'
-    description = "Invalid Credentials."
-    code = 400

api/controllers/console/app/model_config.py

@@ -25,7 +25,6 @@ class ModelConfigResource(Resource):
     @account_initialization_required
     @get_app_model(mode=[AppMode.AGENT_CHAT, AppMode.CHAT, AppMode.COMPLETION])
     def post(self, app_model):
-        
         """Modify app model config"""
         # validate config
         model_configuration = AppModelConfigService.validate_configuration(

api/controllers/console/app/ops_trace.py

@@ -1,101 +0,0 @@
-from flask_restful import Resource, reqparse
-
-from controllers.console import api
-from controllers.console.app.error import TracingConfigCheckError, TracingConfigIsExist, TracingConfigNotExist
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
-from libs.login import login_required
-from services.ops_service import OpsService
-
-
-class TraceAppConfigApi(Resource):
-    """
-    Manage trace app configurations
-    """
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def get(self, app_id):
-        parser = reqparse.RequestParser()
-        parser.add_argument('tracing_provider', type=str, required=True, location='args')
-        args = parser.parse_args()
-
-        try:
-            trace_config = OpsService.get_tracing_app_config(
-                app_id=app_id, tracing_provider=args['tracing_provider']
-                )
-            if not trace_config:
-                return {"has_not_configured": True}
-            return trace_config
-        except Exception as e:
-            raise e
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def post(self, app_id):
-        """Create a new trace app configuration"""
-        parser = reqparse.RequestParser()
-        parser.add_argument('tracing_provider', type=str, required=True, location='json')
-        parser.add_argument('tracing_config', type=dict, required=True, location='json')
-        args = parser.parse_args()
-
-        try:
-            result = OpsService.create_tracing_app_config(
-                app_id=app_id,
-                tracing_provider=args['tracing_provider'],
-                tracing_config=args['tracing_config']
-            )
-            if not result:
-                raise TracingConfigIsExist()
-            if result.get('error'):
-                raise TracingConfigCheckError()
-            return result
-        except Exception as e:
-            raise e
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def patch(self, app_id):
-        """Update an existing trace app configuration"""
-        parser = reqparse.RequestParser()
-        parser.add_argument('tracing_provider', type=str, required=True, location='json')
-        parser.add_argument('tracing_config', type=dict, required=True, location='json')
-        args = parser.parse_args()
-
-        try:
-            result = OpsService.update_tracing_app_config(
-                app_id=app_id,
-                tracing_provider=args['tracing_provider'],
-                tracing_config=args['tracing_config']
-            )
-            if not result:
-                raise TracingConfigNotExist()
-            return {"result": "success"}
-        except Exception as e:
-            raise e
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def delete(self, app_id):
-        """Delete an existing trace app configuration"""
-        parser = reqparse.RequestParser()
-        parser.add_argument('tracing_provider', type=str, required=True, location='args')
-        args = parser.parse_args()
-
-        try:
-            result = OpsService.delete_tracing_app_config(
-                app_id=app_id,
-                tracing_provider=args['tracing_provider']
-            )
-            if not result:
-                raise TracingConfigNotExist()
-            return {"result": "success"}
-        except Exception as e:
-            raise e
-
-
-api.add_resource(TraceAppConfigApi, '/apps/<uuid:app_id>/trace-config')

api/controllers/console/app/site.py

@@ -20,8 +20,6 @@ def parse_app_site_args():
     parser.add_argument('icon_background', type=str, required=False, location='json')
     parser.add_argument('description', type=str, required=False, location='json')
     parser.add_argument('default_language', type=supported_language, required=False, location='json')
-    parser.add_argument('chat_color_theme', type=str, required=False, location='json')
-    parser.add_argument('chat_color_theme_inverted', type=bool, required=False, location='json')
     parser.add_argument('customize_domain', type=str, required=False, location='json')
     parser.add_argument('copyright', type=str, required=False, location='json')
     parser.add_argument('privacy_policy', type=str, required=False, location='json')
@@ -30,7 +28,6 @@ def parse_app_site_args():
                         required=False,
                         location='json')
     parser.add_argument('prompt_public', type=bool, required=False, location='json')
-    parser.add_argument('show_workflow_steps', type=bool, required=False, location='json')
     return parser.parse_args()
 
 
@@ -57,15 +54,12 @@ class AppSite(Resource):
             'icon_background',
             'description',
             'default_language',
-            'chat_color_theme',
-            'chat_color_theme_inverted',
             'customize_domain',
             'copyright',
             'privacy_policy',
             'custom_disclaimer',
             'customize_token_strategy',
-            'prompt_public',
-            'show_workflow_steps'
+            'prompt_public'
         ]:
             value = args.get(attr_name)
             if value is not None:

api/controllers/console/app/workflow.py

@@ -109,34 +109,6 @@ class DraftWorkflowApi(Resource):
         }
 
 
-class DraftWorkflowImportApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
-    @marshal_with(workflow_fields)
-    def post(self, app_model: App):
-        """
-        Import draft workflow
-        """
-        # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
-            raise Forbidden()
-
-        parser = reqparse.RequestParser()
-        parser.add_argument('data', type=str, required=True, nullable=False, location='json')
-        args = parser.parse_args()
-
-        workflow_service = WorkflowService()
-        workflow = workflow_service.import_draft_workflow(
-            app_model=app_model,
-            data=args['data'],
-            account=current_user
-        )
-
-        return workflow
-
-
 class AdvancedChatDraftWorkflowRunApi(Resource):
     @setup_required
     @login_required
@@ -467,7 +439,6 @@ class ConvertToWorkflowApi(Resource):
 
 
 api.add_resource(DraftWorkflowApi, '/apps/<uuid:app_id>/workflows/draft')
-api.add_resource(DraftWorkflowImportApi, '/apps/<uuid:app_id>/workflows/draft/import')
 api.add_resource(AdvancedChatDraftWorkflowRunApi, '/apps/<uuid:app_id>/advanced-chat/workflows/draft/run')
 api.add_resource(DraftWorkflowRunApi, '/apps/<uuid:app_id>/workflows/draft/run')
 api.add_resource(WorkflowTaskStopApi, '/apps/<uuid:app_id>/workflow-runs/tasks/<string:task_id>/stop')

api/controllers/console/auth/login.py

@@ -1,5 +1,3 @@
-from typing import cast
-
 import flask_login
 from flask import current_app, request
 from flask_restful import Resource, reqparse
@@ -7,9 +5,8 @@ from flask_restful import Resource, reqparse
 import services
 from controllers.console import api
 from controllers.console.setup import setup_required
-from libs.helper import email, get_remote_ip
+from libs.helper import email
 from libs.password import valid_password
-from models.account import Account
 from services.account_service import AccountService, TenantService
 
 
@@ -37,7 +34,10 @@ class LoginApi(Resource):
         if len(tenants) == 0:
             return {'result': 'fail', 'data': 'workspace not found, please contact system admin to invite you to join in a workspace'}
 
-        token = AccountService.login(account, ip_address=get_remote_ip(request))
+        AccountService.update_last_login(account, request)
+
+        # todo: return the user info
+        token = AccountService.get_account_jwt_token(account)
 
         return {'result': 'success', 'data': token}
 
@@ -46,9 +46,6 @@ class LogoutApi(Resource):
 
     @setup_required
     def get(self):
-        account = cast(Account, flask_login.current_user)
-        token = request.headers.get('Authorization', '').split(' ')[1]
-        AccountService.logout(account=account, token=token)
         flask_login.logout_user()
         return {'result': 'success'}
 

api/controllers/console/auth/oauth.py

@@ -8,7 +8,6 @@ from flask_restful import Resource
 
 from constants.languages import languages
 from extensions.ext_database import db
-from libs.helper import get_remote_ip
 from libs.oauth import GitHubOAuth, GoogleOAuth, OAuthUserInfo
 from models.account import Account, AccountStatus
 from services.account_service import AccountService, RegisterService, TenantService
@@ -79,7 +78,9 @@ class OAuthCallback(Resource):
 
         TenantService.create_owner_tenant_if_not_exist(account)
 
-        token = AccountService.login(account, ip_address=get_remote_ip(request))
+        AccountService.update_last_login(account, request)
+
+        token = AccountService.get_account_jwt_token(account)
 
         return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?console_token={token}')
 

api/controllers/console/datasets/datasets.py

@@ -8,7 +8,7 @@ import services
 from controllers.console import api
 from controllers.console.apikey import api_key_fields, api_key_list
 from controllers.console.app.error import ProviderNotInitializeError
-from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError
+from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
@@ -17,7 +17,6 @@ from core.model_runtime.entities.model_entities import ModelType
 from core.provider_manager import ProviderManager
 from core.rag.datasource.vdb.vector_type import VectorType
 from core.rag.extractor.entity.extract_setting import ExtractSetting
-from core.rag.retrieval.retrival_methods import RetrievalMethod
 from extensions.ext_database import db
 from fields.app_fields import related_app_list
 from fields.dataset_fields import dataset_detail_fields, dataset_query_detail_fields
@@ -226,15 +225,6 @@ class DatasetApi(Resource):
         except services.errors.dataset.DatasetInUseError:
             raise DatasetInUseError()
 
-class DatasetUseCheckApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def get(self, dataset_id):
-        dataset_id_str = str(dataset_id)
-
-        dataset_is_using = DatasetService.dataset_use_check(dataset_id_str)
-        return {'is_using': dataset_is_using}, 200
 
 class DatasetQueryApi(Resource):
 
@@ -355,8 +345,6 @@ class DatasetIndexingEstimateApi(Resource):
                 "in the Settings -> Model Provider.")
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
-        except Exception as e:
-            raise IndexingEstimateError(str(e))
 
         return response, 200
 
@@ -509,18 +497,16 @@ class DatasetRetrievalSettingApi(Resource):
     def get(self):
         vector_type = current_app.config['VECTOR_STORE']
         match vector_type:
-            case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE:
+            case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT:
                 return {
                     'retrieval_method': [
-                        RetrievalMethod.SEMANTIC_SEARCH
+                        'semantic_search'
                     ]
                 }
-            case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH:
+            case VectorType.QDRANT | VectorType.WEAVIATE:
                 return {
                     'retrieval_method': [
-                        RetrievalMethod.SEMANTIC_SEARCH,
-                        RetrievalMethod.FULL_TEXT_SEARCH,
-                        RetrievalMethod.HYBRID_SEARCH,
+                        'semantic_search', 'full_text_search', 'hybrid_search'
                     ]
                 }
             case _:
@@ -533,18 +519,16 @@ class DatasetRetrievalSettingMockApi(Resource):
     @account_initialization_required
     def get(self, vector_type):
         match vector_type:
-            case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE:
+            case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCEN:
                 return {
                     'retrieval_method': [
-                        RetrievalMethod.SEMANTIC_SEARCH
+                        'semantic_search'
                     ]
                 }
-            case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH:
+            case VectorType.QDRANT | VectorType.WEAVIATE:
                 return {
                     'retrieval_method': [
-                        RetrievalMethod.SEMANTIC_SEARCH,
-                        RetrievalMethod.FULL_TEXT_SEARCH,
-                        RetrievalMethod.HYBRID_SEARCH,
+                        'semantic_search', 'full_text_search', 'hybrid_search'
                     ]
                 }
             case _:
@@ -571,7 +555,6 @@ class DatasetErrorDocs(Resource):
 
 api.add_resource(DatasetListApi, '/datasets')
 api.add_resource(DatasetApi, '/datasets/<uuid:dataset_id>')
-api.add_resource(DatasetUseCheckApi, '/datasets/<uuid:dataset_id>/use-check')
 api.add_resource(DatasetQueryApi, '/datasets/<uuid:dataset_id>/queries')
 api.add_resource(DatasetErrorDocs, '/datasets/<uuid:dataset_id>/error-docs')
 api.add_resource(DatasetIndexingEstimateApi, '/datasets/indexing-estimate')

api/controllers/console/datasets/datasets_document.py

@@ -20,7 +20,6 @@ from controllers.console.datasets.error import (
     ArchivedDocumentImmutableError,
     DocumentAlreadyFinishedError,
     DocumentIndexingError,
-    IndexingEstimateError,
     InvalidActionError,
     InvalidMetadataError,
 )
@@ -389,8 +388,6 @@ class DocumentIndexingEstimateApi(DocumentResource):
                         "in the Settings -> Model Provider.")
                 except ProviderTokenNotInitError as ex:
                     raise ProviderNotInitializeError(ex.description)
-                except Exception as e:
-                    raise IndexingEstimateError(str(e))
 
         return response
 
@@ -496,8 +493,6 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
                     "in the Settings -> Model Provider.")
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
-            except Exception as e:
-                raise IndexingEstimateError(str(e))
         return response
 
 

api/controllers/console/datasets/error.py

@@ -83,9 +83,3 @@ class DatasetInUseError(BaseHTTPException):
     error_code = 'dataset_in_use'
     description = "The dataset is being used by some apps. Please remove the dataset from the apps before deleting it."
     code = 409
-
-
-class IndexingEstimateError(BaseHTTPException):
-    error_code = 'indexing_estimate_error'
-    description = "Knowledge indexing estimate failed: {message}"
-    code = 500

api/controllers/console/datasets/file.py

@@ -14,7 +14,7 @@ from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from fields.file_fields import file_fields, upload_config_fields
 from libs.login import login_required
-from services.file_service import ALLOWED_EXTENSIONS, UNSTRUCTURED_ALLOWED_EXTENSIONS, FileService
+from services.file_service import ALLOWED_EXTENSIONS, UNSTRUSTURED_ALLOWED_EXTENSIONS, FileService
 
 PREVIEW_WORDS_LIMIT = 3000
 
@@ -77,7 +77,7 @@ class FileSupportTypeApi(Resource):
     @account_initialization_required
     def get(self):
         etl_type = current_app.config['ETL_TYPE']
-        allowed_extensions = UNSTRUCTURED_ALLOWED_EXTENSIONS if etl_type == 'Unstructured' else ALLOWED_EXTENSIONS
+        allowed_extensions = UNSTRUSTURED_ALLOWED_EXTENSIONS if etl_type == 'Unstructured' else ALLOWED_EXTENSIONS
         return {'allowed_extensions': allowed_extensions}
 
 

api/controllers/console/setup.py

@@ -3,10 +3,11 @@ from functools import wraps
 from flask import current_app, request
 from flask_restful import Resource, reqparse
 
-from libs.helper import email, get_remote_ip, str_len
+from extensions.ext_database import db
+from libs.helper import email, str_len
 from libs.password import valid_password
 from models.model import DifySetup
-from services.account_service import RegisterService, TenantService
+from services.account_service import AccountService, RegisterService, TenantService
 
 from . import api
 from .error import AlreadySetupError, NotInitValidateError, NotSetupError
@@ -50,17 +51,28 @@ class SetupApi(Resource):
                             required=True, location='json')
         args = parser.parse_args()
 
-        # setup
-        RegisterService.setup(
+        # Register
+        account = RegisterService.register(
             email=args['email'],
             name=args['name'],
-            password=args['password'],
-            ip_address=get_remote_ip(request)
+            password=args['password']
         )
 
+        TenantService.create_owner_tenant_if_not_exist(account)
+
+        setup()
+        AccountService.update_last_login(account, request)
+
         return {'result': 'success'}, 201
 
 
+def setup():
+    dify_setup = DifySetup(
+        version=current_app.config['CURRENT_VERSION']
+    )
+    db.session.add(dify_setup)
+
+
 def setup_required(view):
     @wraps(view)
     def decorated(*args, **kwargs):

api/controllers/console/workspace/tool_providers.py

@@ -104,7 +104,7 @@ class ToolBuiltinProviderIconApi(Resource):
     @setup_required
     def get(self, provider):
         icon_bytes, mimetype = BuiltinToolManageService.get_builtin_tool_provider_icon(provider)
-        icon_cache_max_age = current_app.config.get('TOOL_ICON_CACHE_MAX_AGE')
+        icon_cache_max_age = int(current_app.config.get('TOOL_ICON_CACHE_MAX_AGE'))
         return send_file(io.BytesIO(icon_bytes), mimetype=mimetype, max_age=icon_cache_max_age)
 
 class ToolApiProviderAddApi(Resource):

api/controllers/web/site.py

@@ -26,8 +26,6 @@ class AppSiteApi(WebApiResource):
 
     site_fields = {
         'title': fields.String,
-        'chat_color_theme': fields.String,
-        'chat_color_theme_inverted': fields.Boolean,
         'icon': fields.String,
         'icon_background': fields.String,
         'description': fields.String,
@@ -35,8 +33,7 @@ class AppSiteApi(WebApiResource):
         'privacy_policy': fields.String,
         'custom_disclaimer': fields.String,
         'default_language': fields.String,
-        'prompt_public': fields.Boolean,
-        'show_workflow_steps': fields.Boolean,
+        'prompt_public': fields.Boolean
     }
 
     app_fields = {

api/core/agent/base_agent_runner.py

@@ -32,6 +32,7 @@ from core.model_runtime.entities.model_entities import ModelFeature
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.model_runtime.utils.encoders import jsonable_encoder
 from core.tools.entities.tool_entities import (
+    ToolInvokeMessage,
     ToolParameter,
     ToolRuntimeVariablePool,
 )
@@ -140,6 +141,24 @@ class BaseAgentRunner(AppRunner):
             app_generate_entity.app_config.prompt_template.simple_prompt_template = ''
 
         return app_generate_entity
+
+    def _convert_tool_response_to_str(self, tool_response: list[ToolInvokeMessage]) -> str:
+        """
+        Handle tool response
+        """
+        result = ''
+        for response in tool_response:
+            if response.type == ToolInvokeMessage.MessageType.TEXT:
+                result += response.message
+            elif response.type == ToolInvokeMessage.MessageType.LINK:
+                result += f"result link: {response.message}. please tell user to check it."
+            elif response.type == ToolInvokeMessage.MessageType.IMAGE_LINK or \
+                 response.type == ToolInvokeMessage.MessageType.IMAGE:
+                result += "image has been created and sent to user already, you do not need to create it, just tell the user to check it now."
+            else:
+                result += f"tool response: {response.message}."
+
+        return result
     
     def _convert_tool_to_prompt_message_tool(self, tool: AgentToolEntity) -> tuple[PromptMessageTool, Tool]:
         """

api/core/agent/cot_agent_runner.py

@@ -1,7 +1,7 @@
 import json
 from abc import ABC, abstractmethod
 from collections.abc import Generator
-from typing import Optional, Union
+from typing import Union
 
 from core.agent.base_agent_runner import BaseAgentRunner
 from core.agent.entities import AgentScratchpadUnit
@@ -15,7 +15,6 @@ from core.model_runtime.entities.message_entities import (
     ToolPromptMessage,
     UserPromptMessage,
 )
-from core.ops.ops_trace_manager import TraceQueueManager
 from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransform
 from core.tools.entities.tool_entities import ToolInvokeMeta
 from core.tools.tool.tool import Tool
@@ -43,8 +42,6 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         self._repack_app_generate_entity(app_generate_entity)
         self._init_react_state(query)
 
-        trace_manager = app_generate_entity.trace_manager
-
         # check model mode
         if 'Observation' not in app_generate_entity.model_conf.stop:
             if app_generate_entity.model_conf.provider not in self._ignore_observation_providers:
@@ -214,8 +211,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                     tool_invoke_response, tool_invoke_meta = self._handle_invoke_action(
                         action=scratchpad.action,
                         tool_instances=tool_instances,
-                        message_file_ids=message_file_ids,
-                        trace_manager=trace_manager,
+                        message_file_ids=message_file_ids
                     )
                     scratchpad.observation = tool_invoke_response
                     scratchpad.agent_response = tool_invoke_response
@@ -241,7 +237,8 @@ class CotAgentRunner(BaseAgentRunner, ABC):
 
                 # update prompt tool message
                 for prompt_tool in self._prompt_messages_tools:
-                    self.update_prompt_message_tool(tool_instances[prompt_tool.name], prompt_tool)
+                    self.update_prompt_message_tool(
+                        tool_instances[prompt_tool.name], prompt_tool)
 
             iteration_step += 1
 
@@ -278,15 +275,14 @@ class CotAgentRunner(BaseAgentRunner, ABC):
             message=AssistantPromptMessage(
                 content=final_answer
             ),
-            usage=llm_usage['usage'] if llm_usage['usage'] else LLMUsage.empty_usage(),
+            usage=llm_usage['usage'] if llm_usage['usage'] else LLMUsage.empty_usage(
+            ),
             system_fingerprint=''
         )), PublishFrom.APPLICATION_MANAGER)
 
     def _handle_invoke_action(self, action: AgentScratchpadUnit.Action,
                               tool_instances: dict[str, Tool],
-                              message_file_ids: list[str],
-                              trace_manager: Optional[TraceQueueManager] = None
-                              ) -> tuple[str, ToolInvokeMeta]:
+                              message_file_ids: list[str]) -> tuple[str, ToolInvokeMeta]:
         """
         handle invoke action
         :param action: action
@@ -316,22 +312,21 @@ class CotAgentRunner(BaseAgentRunner, ABC):
             tenant_id=self.tenant_id,
             message=self.message,
             invoke_from=self.application_generate_entity.invoke_from,
-            agent_tool_callback=self.agent_callback,
-            trace_manager=trace_manager,
+            agent_tool_callback=self.agent_callback
         )
 
         # publish files
-        for message_file_id, save_as in message_files:
+        for message_file, save_as in message_files:
             if save_as:
                 self.variables_pool.set_file(
-                    tool_name=tool_call_name, value=message_file_id, name=save_as)
+                    tool_name=tool_call_name, value=message_file.id, name=save_as)
 
             # publish message file
             self.queue_manager.publish(QueueMessageFileEvent(
-                message_file_id=message_file_id
+                message_file_id=message_file.id
             ), PublishFrom.APPLICATION_MANAGER)
             # add message file ids
-            message_file_ids.append(message_file_id)
+            message_file_ids.append(message_file.id)
 
         return tool_invoke_response, tool_invoke_meta
 

api/core/agent/fc_agent_runner.py

@@ -50,9 +50,6 @@ class FunctionCallAgentRunner(BaseAgentRunner):
         }
         final_answer = ''
 
-        # get tracing instance
-        trace_manager = app_generate_entity.trace_manager
-        
         def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage):
             if not final_llm_usage_dict['usage']:
                 final_llm_usage_dict['usage'] = usage
@@ -246,19 +243,18 @@ class FunctionCallAgentRunner(BaseAgentRunner):
                         message=self.message,
                         invoke_from=self.application_generate_entity.invoke_from,
                         agent_tool_callback=self.agent_callback,
-                        trace_manager=trace_manager,
                     )
                     # publish files
-                    for message_file_id, save_as in message_files:
+                    for message_file, save_as in message_files:
                         if save_as:
-                            self.variables_pool.set_file(tool_name=tool_call_name, value=message_file_id, name=save_as)
+                            self.variables_pool.set_file(tool_name=tool_call_name, value=message_file.id, name=save_as)
 
                         # publish message file
                         self.queue_manager.publish(QueueMessageFileEvent(
-                            message_file_id=message_file_id
+                            message_file_id=message_file.id
                         ), PublishFrom.APPLICATION_MANAGER)
                         # add message file ids
-                        message_file_ids.append(message_file_id)
+                        message_file_ids.append(message_file.id)
                     
                     tool_response = {
                         "tool_call_id": tool_call_id,

api/core/app/app_config/easy_ui_based_app/agent/manager.py

@@ -40,7 +40,7 @@ class AgentConfigManager:
                         'provider_type': tool['provider_type'],
                         'provider_id': tool['provider_id'],
                         'tool_name': tool['tool_name'],
-                        'tool_parameters': tool.get('tool_parameters', {})
+                        'tool_parameters': tool['tool_parameters'] if 'tool_parameters' in tool else {}
                     }
 
                     agent_tools.append(AgentToolEntity(**agent_tool_properties))

api/core/app/app_config/entities.py

@@ -183,14 +183,6 @@ class TextToSpeechEntity(BaseModel):
     language: Optional[str] = None
 
 
-class TracingConfigEntity(BaseModel):
-    """
-    Tracing Config Entity.
-    """
-    enabled: bool
-    tracing_provider: str
-
-
 class FileExtraConfig(BaseModel):
     """
     File Upload Entity.
@@ -207,7 +199,7 @@ class AppAdditionalFeatures(BaseModel):
     more_like_this: bool = False
     speech_to_text: bool = False
     text_to_speech: Optional[TextToSpeechEntity] = None
-    trace_config: Optional[TracingConfigEntity] = None
+
 
 class AppConfig(BaseModel):
     """

api/core/app/apps/advanced_chat/app_generator.py

@@ -20,7 +20,6 @@ from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity,
 from core.app.entities.task_entities import ChatbotAppBlockingResponse, ChatbotAppStreamResponse
 from core.file.message_file_parser import MessageFileParser
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
-from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
 from models.account import Account
 from models.model import App, Conversation, EndUser, Message
@@ -30,14 +29,13 @@ logger = logging.getLogger(__name__)
 
 
 class AdvancedChatAppGenerator(MessageBasedAppGenerator):
-    def generate(
-        self, app_model: App,
-        workflow: Workflow,
-        user: Union[Account, EndUser],
-        args: dict,
-        invoke_from: InvokeFrom,
-        stream: bool = True,
-    ) -> Union[dict, Generator[dict, None, None]]:
+    def generate(self, app_model: App,
+                 workflow: Workflow,
+                 user: Union[Account, EndUser],
+                 args: dict,
+                 invoke_from: InvokeFrom,
+                 stream: bool = True) \
+            -> Union[dict, Generator[dict, None, None]]:
         """
         Generate App response.
 
@@ -59,7 +57,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
         inputs = args['inputs']
 
         extras = {
-            "auto_generate_conversation_name": args.get('auto_generate_name', False)
+            "auto_generate_conversation_name": args['auto_generate_name'] if 'auto_generate_name' in args else False
         }
 
         # get conversation
@@ -86,13 +84,6 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             workflow=workflow
         )
 
-        # get tracing instance
-        trace_manager = TraceQueueManager(app_id=app_model.id)
-
-        if invoke_from == InvokeFrom.DEBUGGER:
-            # always enable retriever resource in debugger mode
-            app_config.additional_features.show_retrieve_source = True
-
         # init application generate entity
         application_generate_entity = AdvancedChatAppGenerateEntity(
             task_id=str(uuid.uuid4()),
@@ -104,8 +95,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             user_id=user.id,
             stream=stream,
             invoke_from=invoke_from,
-            extras=extras,
-            trace_manager=trace_manager
+            extras=extras
         )
 
         return self._generate(

api/core/app/apps/advanced_chat/app_runner.py

@@ -70,8 +70,7 @@ class AdvancedChatAppRunner(AppRunner):
                 app_record=app_record,
                 app_generate_entity=application_generate_entity,
                 inputs=inputs,
-                query=query,
-                message_id=message.id
+                query=query
         ):
             return
 
@@ -157,14 +156,11 @@ class AdvancedChatAppRunner(AppRunner):
         # return workflow
         return workflow
 
-    def handle_input_moderation(
-            self, queue_manager: AppQueueManager,
-            app_record: App,
-            app_generate_entity: AdvancedChatAppGenerateEntity,
-            inputs: dict,
-            query: str,
-            message_id: str
-    ) -> bool:
+    def handle_input_moderation(self, queue_manager: AppQueueManager,
+                                app_record: App,
+                                app_generate_entity: AdvancedChatAppGenerateEntity,
+                                inputs: dict,
+                                query: str) -> bool:
         """
         Handle input moderation
         :param queue_manager: application queue manager
@@ -172,7 +168,6 @@ class AdvancedChatAppRunner(AppRunner):
         :param app_generate_entity: application generate entity
         :param inputs: inputs
         :param query: query
-        :param message_id: message id
         :return:
         """
         try:
@@ -183,7 +178,6 @@ class AdvancedChatAppRunner(AppRunner):
                 app_generate_entity=app_generate_entity,
                 inputs=inputs,
                 query=query,
-                message_id=message_id,
             )
         except ModerationException as e:
             self._stream_output(

api/core/app/apps/advanced_chat/generate_task_pipeline.py

@@ -42,7 +42,6 @@ from core.app.task_pipeline.workflow_cycle_manage import WorkflowCycleManage
 from core.file.file_obj import FileVar
 from core.model_runtime.entities.llm_entities import LLMUsage
 from core.model_runtime.utils.encoders import jsonable_encoder
-from core.ops.ops_trace_manager import TraceQueueManager
 from core.workflow.entities.node_entities import NodeType, SystemVariable
 from core.workflow.nodes.answer.answer_node import AnswerNode
 from core.workflow.nodes.answer.entities import TextGenerateRouteChunk, VarGenerateRouteChunk
@@ -70,15 +69,13 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
     _workflow_system_variables: dict[SystemVariable, Any]
     _iteration_nested_relations: dict[str, list[str]]
 
-    def __init__(
-        self, application_generate_entity: AdvancedChatAppGenerateEntity,
-        workflow: Workflow,
-        queue_manager: AppQueueManager,
-        conversation: Conversation,
-        message: Message,
-        user: Union[Account, EndUser],
-        stream: bool
-    ) -> None:
+    def __init__(self, application_generate_entity: AdvancedChatAppGenerateEntity,
+                 workflow: Workflow,
+                 queue_manager: AppQueueManager,
+                 conversation: Conversation,
+                 message: Message,
+                 user: Union[Account, EndUser],
+                 stream: bool) -> None:
         """
         Initialize AdvancedChatAppGenerateTaskPipeline.
         :param application_generate_entity: application generate entity
@@ -129,16 +126,14 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
             self._application_generate_entity.query
         )
 
-        generator = self._process_stream_response(
-            trace_manager=self._application_generate_entity.trace_manager
-        )
+        generator = self._process_stream_response()
         if self._stream:
             return self._to_stream_response(generator)
         else:
             return self._to_blocking_response(generator)
 
     def _to_blocking_response(self, generator: Generator[StreamResponse, None, None]) \
-        -> ChatbotAppBlockingResponse:
+            -> ChatbotAppBlockingResponse:
         """
         Process blocking response.
         :return:
@@ -169,7 +164,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
         raise Exception('Queue listening stopped unexpectedly.')
 
     def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) \
-        -> Generator[ChatbotAppStreamResponse, None, None]:
+            -> Generator[ChatbotAppStreamResponse, None, None]:
         """
         To stream response.
         :return:
@@ -182,9 +177,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                 stream_response=stream_response
             )
 
-    def _process_stream_response(
-        self, trace_manager: Optional[TraceQueueManager] = None
-    ) -> Generator[StreamResponse, None, None]:
+    def _process_stream_response(self) -> Generator[StreamResponse, None, None]:
         """
         Process stream response.
         :return:
@@ -256,9 +249,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                 yield self._handle_iteration_to_stream_response(self._application_generate_entity.task_id, event)
                 self._handle_iteration_operation(event)
             elif isinstance(event, QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent):
-                workflow_run = self._handle_workflow_finished(
-                    event, conversation_id=self._conversation.id, trace_manager=trace_manager
-                )
+                workflow_run = self._handle_workflow_finished(event)
                 if workflow_run:
                     yield self._workflow_finish_to_stream_response(
                         task_id=self._application_generate_entity.task_id,
@@ -301,7 +292,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                     continue
 
                 if not self._is_stream_out_support(
-                    event=event
+                        event=event
                 ):
                     continue
 
@@ -370,7 +361,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
             id=self._message.id,
             **extras
         )
-
+    
     def _get_stream_generate_routes(self) -> dict[str, ChatflowStreamGenerateRoute]:
         """
         Get stream generate routes.
@@ -400,9 +391,9 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                 )
 
         return stream_generate_routes
-
+    
     def _get_answer_start_at_node_ids(self, graph: dict, target_node_id: str) \
-        -> list[str]:
+            -> list[str]:
         """
         Get answer start at node id.
         :param graph: graph
@@ -423,14 +414,14 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
             target_node = next((node for node in nodes if node.get('id') == target_node_id), None)
             if not target_node:
                 return []
-
+            
             node_iteration_id = target_node.get('data', {}).get('iteration_id')
             # get iteration start node id
             for node in nodes:
                 if node.get('id') == node_iteration_id:
                     if node.get('data', {}).get('start_node_id') == target_node_id:
                         return [target_node_id]
-
+                    
             return []
 
         start_node_ids = []
@@ -466,7 +457,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                     start_node_ids.extend(sub_start_node_ids)
 
         return start_node_ids
-
+    
     def _get_iteration_nested_relations(self, graph: dict) -> dict[str, list[str]]:
         """
         Get iteration nested relations.
@@ -475,18 +466,18 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
         """
         nodes = graph.get('nodes')
 
-        iteration_ids = [node.get('id') for node in nodes
+        iteration_ids = [node.get('id') for node in nodes 
                          if node.get('data', {}).get('type') in [
                              NodeType.ITERATION.value,
                              NodeType.LOOP.value,
-                         ]]
+                        ]]
 
         return {
             iteration_id: [
                 node.get('id') for node in nodes if node.get('data', {}).get('iteration_id') == iteration_id
             ] for iteration_id in iteration_ids
         }
-
+    
     def _generate_stream_outputs_when_node_started(self) -> Generator:
         """
         Generate stream outputs.
@@ -494,8 +485,8 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
         """
         if self._task_state.current_stream_generate_state:
             route_chunks = self._task_state.current_stream_generate_state.generate_route[
-                           self._task_state.current_stream_generate_state.current_route_position:
-                           ]
+                self._task_state.current_stream_generate_state.current_route_position:
+            ]
 
             for route_chunk in route_chunks:
                 if route_chunk.type == 'text':
@@ -515,8 +506,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
 
             # all route chunks are generated
             if self._task_state.current_stream_generate_state.current_route_position == len(
-                self._task_state.current_stream_generate_state.generate_route
-            ):
+                    self._task_state.current_stream_generate_state.generate_route):
                 self._task_state.current_stream_generate_state = None
 
     def _generate_stream_outputs_when_node_finished(self) -> Optional[Generator]:
@@ -529,7 +519,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
 
         route_chunks = self._task_state.current_stream_generate_state.generate_route[
                        self._task_state.current_stream_generate_state.current_route_position:]
-
+        
         for route_chunk in route_chunks:
             if route_chunk.type == 'text':
                 route_chunk = cast(TextGenerateRouteChunk, route_chunk)
@@ -561,8 +551,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                         value = iteration_state.current_index
                     elif value_selector[1] == 'item':
                         value = iterator_selector[iteration_state.current_index] if iteration_state.current_index < len(
-                            iterator_selector
-                        ) else None
+                            iterator_selector) else None
                 else:
                     # check chunk node id is before current node id or equal to current node id
                     if route_chunk_node_id not in self._task_state.ran_node_execution_infos:
@@ -573,15 +562,14 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                     # get route chunk node execution info
                     route_chunk_node_execution_info = self._task_state.ran_node_execution_infos[route_chunk_node_id]
                     if (route_chunk_node_execution_info.node_type == NodeType.LLM
-                        and latest_node_execution_info.node_type == NodeType.LLM):
+                            and latest_node_execution_info.node_type == NodeType.LLM):
                         # only LLM support chunk stream output
                         self._task_state.current_stream_generate_state.current_route_position += 1
                         continue
 
                     # get route chunk node execution
                     route_chunk_node_execution = db.session.query(WorkflowNodeExecution).filter(
-                        WorkflowNodeExecution.id == route_chunk_node_execution_info.workflow_node_execution_id
-                    ).first()
+                        WorkflowNodeExecution.id == route_chunk_node_execution_info.workflow_node_execution_id).first()
 
                     outputs = route_chunk_node_execution.outputs_dict
 
@@ -643,8 +631,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
 
         # all route chunks are generated
         if self._task_state.current_stream_generate_state.current_route_position == len(
-            self._task_state.current_stream_generate_state.generate_route
-        ):
+                self._task_state.current_stream_generate_state.generate_route):
             self._task_state.current_stream_generate_state = None
 
     def _is_stream_out_support(self, event: QueueTextChunkEvent) -> bool:

api/core/app/apps/agent_chat/app_generator.py

@@ -19,7 +19,6 @@ from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueMa
 from core.app.entities.app_invoke_entities import AgentChatAppGenerateEntity, InvokeFrom
 from core.file.message_file_parser import MessageFileParser
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
-from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
 from models.account import Account
 from models.model import App, EndUser
@@ -57,7 +56,7 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
         inputs = args['inputs']
 
         extras = {
-            "auto_generate_conversation_name": args.get('auto_generate_name', True)
+            "auto_generate_conversation_name": args['auto_generate_name'] if 'auto_generate_name' in args else True
         }
 
         # get conversation
@@ -83,11 +82,6 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
                 config=args.get('model_config')
             )
 
-            # always enable retriever resource in debugger mode
-            override_model_config_dict["retriever_resource"] = {
-                "enabled": True
-            }
-
         # parse files
         files = args['files'] if args.get('files') else []
         message_file_parser = MessageFileParser(tenant_id=app_model.tenant_id, app_id=app_model.id)
@@ -109,9 +103,6 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
             override_config_dict=override_model_config_dict
         )
 
-        # get tracing instance
-        trace_manager = TraceQueueManager(app_model.id)
-
         # init application generate entity
         application_generate_entity = AgentChatAppGenerateEntity(
             task_id=str(uuid.uuid4()),
@@ -125,8 +116,7 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
             stream=stream,
             invoke_from=invoke_from,
             extras=extras,
-            call_depth=0,
-            trace_manager=trace_manager
+            call_depth=0
         )
 
         # init generate records
@@ -163,7 +153,7 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
             conversation=conversation,
             message=message,
             user=user,
-            stream=stream,
+            stream=stream
         )
 
         return AgentChatAppGenerateResponseConverter.convert(
@@ -171,13 +161,11 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
             invoke_from=invoke_from
         )
 
-    def _generate_worker(
-        self, flask_app: Flask,
-        application_generate_entity: AgentChatAppGenerateEntity,
-        queue_manager: AppQueueManager,
-        conversation_id: str,
-        message_id: str,
-    ) -> None:
+    def _generate_worker(self, flask_app: Flask,
+                         application_generate_entity: AgentChatAppGenerateEntity,
+                         queue_manager: AppQueueManager,
+                         conversation_id: str,
+                         message_id: str) -> None:
         """
         Generate worker in a new thread.
         :param flask_app: Flask app
@@ -199,7 +187,7 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
                     application_generate_entity=application_generate_entity,
                     queue_manager=queue_manager,
                     conversation=conversation,
-                    message=message,
+                    message=message
                 )
             except GenerateTaskStoppedException:
                 pass

api/core/app/apps/agent_chat/app_runner.py

@@ -28,13 +28,10 @@ class AgentChatAppRunner(AppRunner):
     """
     Agent Application Runner
     """
-
-    def run(
-        self, application_generate_entity: AgentChatAppGenerateEntity,
-        queue_manager: AppQueueManager,
-        conversation: Conversation,
-        message: Message,
-    ) -> None:
+    def run(self, application_generate_entity: AgentChatAppGenerateEntity,
+            queue_manager: AppQueueManager,
+            conversation: Conversation,
+            message: Message) -> None:
         """
         Run assistant application
         :param application_generate_entity: application generate entity
@@ -103,7 +100,6 @@ class AgentChatAppRunner(AppRunner):
                 app_generate_entity=application_generate_entity,
                 inputs=inputs,
                 query=query,
-                message_id=message.id
             )
         except ModerationException as e:
             self.direct_output(
@@ -203,7 +199,7 @@ class AgentChatAppRunner(AppRunner):
         llm_model = cast(LargeLanguageModel, model_instance.model_type_instance)
         model_schema = llm_model.get_model_schema(model_instance.model, model_instance.credentials)
 
-        if {ModelFeature.MULTI_TOOL_CALL, ModelFeature.TOOL_CALL}.intersection(model_schema.features or []):
+        if set([ModelFeature.MULTI_TOOL_CALL, ModelFeature.TOOL_CALL]).intersection(model_schema.features or []):
             agent_entity.strategy = AgentEntity.Strategy.FUNCTION_CALLING
 
         conversation = db.session.query(Conversation).filter(Conversation.id == conversation.id).first()
@@ -223,7 +219,7 @@ class AgentChatAppRunner(AppRunner):
             runner_cls = FunctionCallAgentRunner
         else:
             raise ValueError(f"Invalid agent strategy: {agent_entity.strategy}")
-
+        
         runner = runner_cls(
             tenant_id=app_config.tenant_id,
             application_generate_entity=application_generate_entity,

api/core/app/apps/base_app_runner.py

@@ -338,14 +338,11 @@ class AppRunner:
             ), PublishFrom.APPLICATION_MANAGER
         )
 
-    def moderation_for_inputs(
-            self, app_id: str,
-            tenant_id: str,
-            app_generate_entity: AppGenerateEntity,
-            inputs: dict,
-            query: str,
-            message_id: str,
-    ) -> tuple[bool, dict, str]:
+    def moderation_for_inputs(self, app_id: str,
+                              tenant_id: str,
+                              app_generate_entity: AppGenerateEntity,
+                              inputs: dict,
+                              query: str) -> tuple[bool, dict, str]:
         """
         Process sensitive_word_avoidance.
         :param app_id: app id
@@ -353,7 +350,6 @@ class AppRunner:
         :param app_generate_entity: app generate entity
         :param inputs: inputs
         :param query: query
-        :param message_id: message id
         :return:
         """
         moderation_feature = InputModeration()
@@ -362,9 +358,7 @@ class AppRunner:
             tenant_id=tenant_id,
             app_config=app_generate_entity.app_config,
             inputs=inputs,
-            query=query if query else '',
-            message_id=message_id,
-            trace_manager=app_generate_entity.trace_manager
+            query=query if query else ''
         )
     
     def check_hosting_moderation(self, application_generate_entity: EasyUIBasedAppGenerateEntity,

api/core/app/apps/chat/app_config_manager.py

@@ -50,9 +50,6 @@ class ChatAppConfigManager(BaseAppConfigManager):
             app_model_config_dict = app_model_config.to_dict()
             config_dict = app_model_config_dict.copy()
         else:
-            if not override_config_dict:
-                raise Exception('override_config_dict is required when config_from is ARGS')
-
             config_dict = override_config_dict
 
         app_mode = AppMode.value_of(app_model.mode)

api/core/app/apps/chat/app_generator.py

@@ -19,7 +19,6 @@ from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueMa
 from core.app.entities.app_invoke_entities import ChatAppGenerateEntity, InvokeFrom
 from core.file.message_file_parser import MessageFileParser
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
-from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
 from models.account import Account
 from models.model import App, EndUser
@@ -28,13 +27,12 @@ logger = logging.getLogger(__name__)
 
 
 class ChatAppGenerator(MessageBasedAppGenerator):
-    def generate(
-        self, app_model: App,
-        user: Union[Account, EndUser],
-        args: Any,
-        invoke_from: InvokeFrom,
-        stream: bool = True,
-    ) -> Union[dict, Generator[dict, None, None]]:
+    def generate(self, app_model: App,
+                 user: Union[Account, EndUser],
+                 args: Any,
+                 invoke_from: InvokeFrom,
+                 stream: bool = True) \
+            -> Union[dict, Generator[dict, None, None]]:
         """
         Generate App response.
 
@@ -55,7 +53,7 @@ class ChatAppGenerator(MessageBasedAppGenerator):
         inputs = args['inputs']
 
         extras = {
-            "auto_generate_conversation_name": args.get('auto_generate_name', True)
+            "auto_generate_conversation_name": args['auto_generate_name'] if 'auto_generate_name' in args else True
         }
 
         # get conversation
@@ -81,11 +79,6 @@ class ChatAppGenerator(MessageBasedAppGenerator):
                 config=args.get('model_config')
             )
 
-            # always enable retriever resource in debugger mode
-            override_model_config_dict["retriever_resource"] = {
-                "enabled": True
-            }
-
         # parse files
         files = args['files'] if args.get('files') else []
         message_file_parser = MessageFileParser(tenant_id=app_model.tenant_id, app_id=app_model.id)
@@ -107,9 +100,6 @@ class ChatAppGenerator(MessageBasedAppGenerator):
             override_config_dict=override_model_config_dict
         )
 
-        # get tracing instance
-        trace_manager = TraceQueueManager(app_model.id)
-
         # init application generate entity
         application_generate_entity = ChatAppGenerateEntity(
             task_id=str(uuid.uuid4()),
@@ -122,8 +112,7 @@ class ChatAppGenerator(MessageBasedAppGenerator):
             user_id=user.id,
             stream=stream,
             invoke_from=invoke_from,
-            extras=extras,
-            trace_manager=trace_manager
+            extras=extras
         )
 
         # init generate records
@@ -160,7 +149,7 @@ class ChatAppGenerator(MessageBasedAppGenerator):
             conversation=conversation,
             message=message,
             user=user,
-            stream=stream,
+            stream=stream
         )
 
         return ChatAppGenerateResponseConverter.convert(

api/core/app/apps/chat/app_runner.py

@@ -96,7 +96,6 @@ class ChatAppRunner(AppRunner):
                 app_generate_entity=application_generate_entity,
                 inputs=inputs,
                 query=query,
-                message_id=message.id
             )
         except ModerationException as e:
             self.direct_output(
@@ -155,7 +154,7 @@ class ChatAppRunner(AppRunner):
                 application_generate_entity.invoke_from
             )
 
-            dataset_retrieval = DatasetRetrieval(application_generate_entity)
+            dataset_retrieval = DatasetRetrieval()
             context = dataset_retrieval.retrieve(
                 app_id=app_record.id,
                 user_id=application_generate_entity.user_id,
@@ -166,8 +165,7 @@ class ChatAppRunner(AppRunner):
                 invoke_from=application_generate_entity.invoke_from,
                 show_retrieve_source=app_config.additional_features.show_retrieve_source,
                 hit_callback=hit_callback,
-                memory=memory,
-                message_id=message.id,
+                memory=memory
             )
 
         # reorganize all inputs and template to prompt messages

api/core/app/apps/completion/app_generator.py

@@ -19,7 +19,6 @@ from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueMa
 from core.app.entities.app_invoke_entities import CompletionAppGenerateEntity, InvokeFrom
 from core.file.message_file_parser import MessageFileParser
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
-from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
 from models.account import Account
 from models.model import App, EndUser, Message
@@ -95,9 +94,6 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
             override_config_dict=override_model_config_dict
         )
 
-        # get tracing instance
-        trace_manager = TraceQueueManager(app_model.id)
-
         # init application generate entity
         application_generate_entity = CompletionAppGenerateEntity(
             task_id=str(uuid.uuid4()),
@@ -109,8 +105,7 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
             user_id=user.id,
             stream=stream,
             invoke_from=invoke_from,
-            extras=extras,
-            trace_manager=trace_manager
+            extras=extras
         )
 
         # init generate records
@@ -146,7 +141,7 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
             conversation=conversation,
             message=message,
             user=user,
-            stream=stream,
+            stream=stream
         )
 
         return CompletionAppGenerateResponseConverter.convert(
@@ -163,6 +158,7 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
         :param flask_app: Flask app
         :param application_generate_entity: application generate entity
         :param queue_manager: queue manager
+        :param conversation_id: conversation ID
         :param message_id: message ID
         :return:
         """
@@ -304,7 +300,7 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
             conversation=conversation,
             message=message,
             user=user,
-            stream=stream,
+            stream=stream
         )
 
         return CompletionAppGenerateResponseConverter.convert(

api/core/app/apps/completion/app_runner.py

@@ -77,7 +77,6 @@ class CompletionAppRunner(AppRunner):
                 app_generate_entity=application_generate_entity,
                 inputs=inputs,
                 query=query,
-                message_id=message.id
             )
         except ModerationException as e:
             self.direct_output(
@@ -115,7 +114,7 @@ class CompletionAppRunner(AppRunner):
             if dataset_config and dataset_config.retrieve_config.query_variable:
                 query = inputs.get(dataset_config.retrieve_config.query_variable, "")
 
-            dataset_retrieval = DatasetRetrieval(application_generate_entity)
+            dataset_retrieval = DatasetRetrieval()
             context = dataset_retrieval.retrieve(
                 app_id=app_record.id,
                 user_id=application_generate_entity.user_id,
@@ -125,8 +124,7 @@ class CompletionAppRunner(AppRunner):
                 query=query,
                 invoke_from=application_generate_entity.invoke_from,
                 show_retrieve_source=app_config.additional_features.show_retrieve_source,
-                hit_callback=hit_callback,
-                message_id=message.id
+                hit_callback=hit_callback
             )
 
         # reorganize all inputs and template to prompt messages

api/core/app/apps/message_based_app_generator.py

@@ -35,23 +35,22 @@ logger = logging.getLogger(__name__)
 
 class MessageBasedAppGenerator(BaseAppGenerator):
 
-    def _handle_response(
-        self, application_generate_entity: Union[
-            ChatAppGenerateEntity,
-            CompletionAppGenerateEntity,
-            AgentChatAppGenerateEntity,
-            AdvancedChatAppGenerateEntity
-        ],
-        queue_manager: AppQueueManager,
-        conversation: Conversation,
-        message: Message,
-        user: Union[Account, EndUser],
-        stream: bool = False,
-    ) -> Union[
-        ChatbotAppBlockingResponse,
-        CompletionAppBlockingResponse,
-        Generator[Union[ChatbotAppStreamResponse, CompletionAppStreamResponse], None, None]
-    ]:
+    def _handle_response(self, application_generate_entity: Union[
+        ChatAppGenerateEntity,
+        CompletionAppGenerateEntity,
+        AgentChatAppGenerateEntity,
+        AdvancedChatAppGenerateEntity
+    ],
+                         queue_manager: AppQueueManager,
+                         conversation: Conversation,
+                         message: Message,
+                         user: Union[Account, EndUser],
+                         stream: bool = False) \
+            -> Union[
+                ChatbotAppBlockingResponse,
+                CompletionAppBlockingResponse,
+                Generator[Union[ChatbotAppStreamResponse, CompletionAppStreamResponse], None, None]
+            ]:
         """
         Handle response.
         :param application_generate_entity: application generate entity

api/core/app/apps/workflow/app_generator.py

@@ -20,7 +20,6 @@ from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerat
 from core.app.entities.task_entities import WorkflowAppBlockingResponse, WorkflowAppStreamResponse
 from core.file.message_file_parser import MessageFileParser
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
-from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
 from models.account import Account
 from models.model import App, EndUser
@@ -30,15 +29,14 @@ logger = logging.getLogger(__name__)
 
 
 class WorkflowAppGenerator(BaseAppGenerator):
-    def generate(
-        self, app_model: App,
-        workflow: Workflow,
-        user: Union[Account, EndUser],
-        args: dict,
-        invoke_from: InvokeFrom,
-        stream: bool = True,
-        call_depth: int = 0,
-    ) -> Union[dict, Generator[dict, None, None]]:
+    def generate(self, app_model: App,
+                 workflow: Workflow,
+                 user: Union[Account, EndUser],
+                 args: dict,
+                 invoke_from: InvokeFrom,
+                 stream: bool = True,
+                 call_depth: int = 0) \
+            -> Union[dict, Generator[dict, None, None]]:
         """
         Generate App response.
 
@@ -48,7 +46,6 @@ class WorkflowAppGenerator(BaseAppGenerator):
         :param args: request args
         :param invoke_from: invoke from source
         :param stream: is stream
-        :param call_depth: call depth
         """
         inputs = args['inputs']
 
@@ -71,9 +68,6 @@ class WorkflowAppGenerator(BaseAppGenerator):
             workflow=workflow
         )
 
-        # get tracing instance
-        trace_manager = TraceQueueManager(app_model.id)
-
         # init application generate entity
         application_generate_entity = WorkflowAppGenerateEntity(
             task_id=str(uuid.uuid4()),
@@ -83,8 +77,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
             user_id=user.id,
             stream=stream,
             invoke_from=invoke_from,
-            call_depth=call_depth,
-            trace_manager=trace_manager
+            call_depth=call_depth
         )
 
         return self._generate(
@@ -94,18 +87,17 @@ class WorkflowAppGenerator(BaseAppGenerator):
             application_generate_entity=application_generate_entity,
             invoke_from=invoke_from,
             stream=stream,
-            call_depth=call_depth,
+            call_depth=call_depth
         )
 
-    def _generate(
-        self, app_model: App,
-        workflow: Workflow,
-        user: Union[Account, EndUser],
-        application_generate_entity: WorkflowAppGenerateEntity,
-        invoke_from: InvokeFrom,
-        stream: bool = True,
-        call_depth: int = 0
-    ) -> Union[dict, Generator[dict, None, None]]:
+    def _generate(self, app_model: App,
+                 workflow: Workflow,
+                 user: Union[Account, EndUser],
+                 application_generate_entity: WorkflowAppGenerateEntity,
+                 invoke_from: InvokeFrom,
+                 stream: bool = True,
+                 call_depth: int = 0) \
+            -> Union[dict, Generator[dict, None, None]]:
         """
         Generate App response.
 
@@ -139,7 +131,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
             workflow=workflow,
             queue_manager=queue_manager,
             user=user,
-            stream=stream,
+            stream=stream
         )
 
         return WorkflowAppGenerateResponseConverter.convert(

api/core/app/apps/workflow/generate_task_pipeline.py

@@ -1,6 +1,6 @@
 import logging
 from collections.abc import Generator
-from typing import Any, Optional, Union
+from typing import Any, Union
 
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import (
@@ -36,7 +36,6 @@ from core.app.entities.task_entities import (
 )
 from core.app.task_pipeline.based_generate_task_pipeline import BasedGenerateTaskPipeline
 from core.app.task_pipeline.workflow_cycle_manage import WorkflowCycleManage
-from core.ops.ops_trace_manager import TraceQueueManager
 from core.workflow.entities.node_entities import NodeType, SystemVariable
 from core.workflow.nodes.end.end_node import EndNode
 from extensions.ext_database import db
@@ -105,9 +104,7 @@ class WorkflowAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCycleMa
         db.session.refresh(self._user)
         db.session.close()
 
-        generator = self._process_stream_response(
-            trace_manager=self._application_generate_entity.trace_manager
-        )
+        generator = self._process_stream_response()
         if self._stream:
             return self._to_stream_response(generator)
         else:
@@ -161,10 +158,7 @@ class WorkflowAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCycleMa
                 stream_response=stream_response
             )
 
-    def _process_stream_response(
-        self,
-        trace_manager: Optional[TraceQueueManager] = None
-    ) -> Generator[StreamResponse, None, None]:
+    def _process_stream_response(self) -> Generator[StreamResponse, None, None]:
         """
         Process stream response.
         :return:
@@ -221,9 +215,7 @@ class WorkflowAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCycleMa
                 yield self._handle_iteration_to_stream_response(self._application_generate_entity.task_id, event)
                 self._handle_iteration_operation(event)
             elif isinstance(event, QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent):
-                workflow_run = self._handle_workflow_finished(
-                    event, trace_manager=trace_manager
-                )
+                workflow_run = self._handle_workflow_finished(event)
 
                 # save workflow app log
                 self._save_workflow_app_log(workflow_run)

api/core/app/entities/app_invoke_entities.py

@@ -7,7 +7,6 @@ from core.app.app_config.entities import AppConfig, EasyUIBasedAppConfig, Workfl
 from core.entities.provider_configuration import ProviderModelBundle
 from core.file.file_obj import FileVar
 from core.model_runtime.entities.model_entities import AIModelEntity
-from core.ops.ops_trace_manager import TraceQueueManager
 
 
 class InvokeFrom(Enum):
@@ -90,12 +89,6 @@ class AppGenerateEntity(BaseModel):
     # extra parameters, like: auto_generate_conversation_name
     extras: dict[str, Any] = {}
 
-    # tracing instance
-    trace_manager: Optional[TraceQueueManager] = None
-
-    class Config:
-        arbitrary_types_allowed = True
-
 
 class EasyUIBasedAppGenerateEntity(AppGenerateEntity):
     """

api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py

@@ -44,7 +44,6 @@ from core.model_runtime.entities.message_entities import (
 )
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.model_runtime.utils.encoders import jsonable_encoder
-from core.ops.ops_trace_manager import TraceQueueManager, TraceTask, TraceTaskName
 from core.prompt.utils.prompt_message_util import PromptMessageUtil
 from core.prompt.utils.prompt_template_parser import PromptTemplateParser
 from events.message_event import message_was_created
@@ -101,9 +100,7 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline, MessageCycleMan
 
         self._conversation_name_generate_thread = None
 
-    def process(
-        self,
-    ) -> Union[
+    def process(self) -> Union[
         ChatbotAppBlockingResponse,
         CompletionAppBlockingResponse,
         Generator[Union[ChatbotAppStreamResponse, CompletionAppStreamResponse], None, None]
@@ -123,9 +120,7 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline, MessageCycleMan
                 self._application_generate_entity.query
             )
 
-        generator = self._process_stream_response(
-            trace_manager=self._application_generate_entity.trace_manager
-        )
+        generator = self._process_stream_response()
         if self._stream:
             return self._to_stream_response(generator)
         else:
@@ -202,9 +197,7 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline, MessageCycleMan
                     stream_response=stream_response
                 )
 
-    def _process_stream_response(
-        self, trace_manager: Optional[TraceQueueManager] = None
-    ) -> Generator[StreamResponse, None, None]:
+    def _process_stream_response(self) -> Generator[StreamResponse, None, None]:
         """
         Process stream response.
         :return:
@@ -231,7 +224,7 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline, MessageCycleMan
                     yield self._message_replace_to_stream_response(answer=output_moderation_answer)
 
                 # Save message
-                self._save_message(trace_manager)
+                self._save_message()
 
                 yield self._message_end_to_stream_response()
             elif isinstance(event, QueueRetrieverResourcesEvent):
@@ -276,9 +269,7 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline, MessageCycleMan
         if self._conversation_name_generate_thread:
             self._conversation_name_generate_thread.join()
 
-    def _save_message(
-        self, trace_manager: Optional[TraceQueueManager] = None
-    ) -> None:
+    def _save_message(self) -> None:
         """
         Save message.
         :return:
@@ -309,15 +300,6 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline, MessageCycleMan
 
         db.session.commit()
 
-        if trace_manager:
-            trace_manager.add_trace_task(
-                TraceTask(
-                    TraceTaskName.MESSAGE_TRACE,
-                    conversation_id=self._conversation.id,
-                    message_id=self._message.id
-                )
-            )
-
         message_was_created.send(
             self._message,
             application_generate_entity=self._application_generate_entity,

api/core/app/task_pipeline/message_cycle_manage.py

@@ -167,11 +167,8 @@ class MessageCycleManage:
                     extension = '.bin'
             else:
                 extension = '.bin'
-            # add sign url to local file
-            if message_file.url.startswith('http'):
-                url = message_file.url
-            else:
-                url = ToolFileManager.sign_file(tool_file_id=tool_file_id, extension=extension)
+            # add sign url
+            url = ToolFileManager.sign_file(tool_file_id=tool_file_id, extension=extension)
 
             return MessageFileStreamResponse(
                 task_id=self._application_generate_entity.task_id,

api/core/app/task_pipeline/workflow_cycle_manage.py

@@ -22,7 +22,6 @@ from core.app.entities.task_entities import (
 from core.app.task_pipeline.workflow_iteration_cycle_manage import WorkflowIterationCycleManage
 from core.file.file_obj import FileVar
 from core.model_runtime.utils.encoders import jsonable_encoder
-from core.ops.ops_trace_manager import TraceQueueManager, TraceTask, TraceTaskName
 from core.tools.tool_manager import ToolManager
 from core.workflow.entities.node_entities import NodeRunMetadataKey, NodeType
 from core.workflow.nodes.tool.entities import ToolNodeData
@@ -95,15 +94,11 @@ class WorkflowCycleManage(WorkflowIterationCycleManage):
 
         return workflow_run
 
-    def _workflow_run_success(
-        self, workflow_run: WorkflowRun,
-        start_at: float,
-        total_tokens: int,
-        total_steps: int,
-        outputs: Optional[str] = None,
-        conversation_id: Optional[str] = None,
-        trace_manager: Optional[TraceQueueManager] = None
-    ) -> WorkflowRun:
+    def _workflow_run_success(self, workflow_run: WorkflowRun,
+                              start_at: float,
+                              total_tokens: int,
+                              total_steps: int,
+                              outputs: Optional[str] = None) -> WorkflowRun:
         """
         Workflow run success
         :param workflow_run: workflow run
@@ -111,7 +106,6 @@ class WorkflowCycleManage(WorkflowIterationCycleManage):
         :param total_tokens: total tokens
         :param total_steps: total steps
         :param outputs: outputs
-        :param conversation_id: conversation id
         :return:
         """
         workflow_run.status = WorkflowRunStatus.SUCCEEDED.value
@@ -125,27 +119,14 @@ class WorkflowCycleManage(WorkflowIterationCycleManage):
         db.session.refresh(workflow_run)
         db.session.close()
 
-        if trace_manager:
-            trace_manager.add_trace_task(
-                TraceTask(
-                    TraceTaskName.WORKFLOW_TRACE,
-                    workflow_run=workflow_run,
-                    conversation_id=conversation_id,
-                )
-            )
-
         return workflow_run
 
-    def _workflow_run_failed(
-        self, workflow_run: WorkflowRun,
-        start_at: float,
-        total_tokens: int,
-        total_steps: int,
-        status: WorkflowRunStatus,
-        error: str,
-        conversation_id: Optional[str] = None,
-        trace_manager: Optional[TraceQueueManager] = None
-    ) -> WorkflowRun:
+    def _workflow_run_failed(self, workflow_run: WorkflowRun,
+                             start_at: float,
+                             total_tokens: int,
+                             total_steps: int,
+                             status: WorkflowRunStatus,
+                             error: str) -> WorkflowRun:
         """
         Workflow run failed
         :param workflow_run: workflow run
@@ -167,15 +148,6 @@ class WorkflowCycleManage(WorkflowIterationCycleManage):
         db.session.refresh(workflow_run)
         db.session.close()
 
-        if trace_manager:
-            trace_manager.add_trace_task(
-                TraceTask(
-                    TraceTaskName.WORKFLOW_TRACE,
-                    workflow_run=workflow_run,
-                    conversation_id=conversation_id,
-                )
-            )
-
         return workflow_run
 
     def _init_node_execution_from_workflow_run(self, workflow_run: WorkflowRun,
@@ -208,8 +180,7 @@ class WorkflowCycleManage(WorkflowIterationCycleManage):
             title=node_title,
             status=WorkflowNodeExecutionStatus.RUNNING.value,
             created_by_role=workflow_run.created_by_role,
-            created_by=workflow_run.created_by,
-            created_at=datetime.now(timezone.utc).replace(tzinfo=None)
+            created_by=workflow_run.created_by
         )
 
         db.session.add(workflow_node_execution)
@@ -469,9 +440,9 @@ class WorkflowCycleManage(WorkflowIterationCycleManage):
         current_node_execution = self._task_state.ran_node_execution_infos[event.node_id]
         workflow_node_execution = db.session.query(WorkflowNodeExecution).filter(
             WorkflowNodeExecution.id == current_node_execution.workflow_node_execution_id).first()
-
+        
         execution_metadata = event.execution_metadata if isinstance(event, QueueNodeSucceededEvent) else None
-
+        
         if self._iteration_state and self._iteration_state.current_iterations:
             if not execution_metadata:
                 execution_metadata = {}
@@ -499,7 +470,7 @@ class WorkflowCycleManage(WorkflowIterationCycleManage):
             if execution_metadata and execution_metadata.get(NodeRunMetadataKey.TOTAL_TOKENS):
                 self._task_state.total_tokens += (
                     int(execution_metadata.get(NodeRunMetadataKey.TOTAL_TOKENS)))
-
+                
                 if self._iteration_state:
                     for iteration_node_id in self._iteration_state.current_iterations:
                         data = self._iteration_state.current_iterations[iteration_node_id]
@@ -525,18 +496,13 @@ class WorkflowCycleManage(WorkflowIterationCycleManage):
 
         return workflow_node_execution
 
-    def _handle_workflow_finished(
-        self, event: QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent,
-        conversation_id: Optional[str] = None,
-        trace_manager: Optional[TraceQueueManager] = None
-    ) -> Optional[WorkflowRun]:
+    def _handle_workflow_finished(self, event: QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent) \
+            -> Optional[WorkflowRun]:
         workflow_run = db.session.query(WorkflowRun).filter(
             WorkflowRun.id == self._task_state.workflow_run_id).first()
         if not workflow_run:
             return None
 
-        if conversation_id is None:
-            conversation_id = self._application_generate_entity.inputs.get('sys.conversation_id')
         if isinstance(event, QueueStopEvent):
             workflow_run = self._workflow_run_failed(
                 workflow_run=workflow_run,
@@ -544,9 +510,7 @@ class WorkflowCycleManage(WorkflowIterationCycleManage):
                 total_tokens=self._task_state.total_tokens,
                 total_steps=self._task_state.total_steps,
                 status=WorkflowRunStatus.STOPPED,
-                error='Workflow stopped.',
-                conversation_id=conversation_id,
-                trace_manager=trace_manager
+                error='Workflow stopped.'
             )
 
             latest_node_execution_info = self._task_state.latest_node_execution_info
@@ -567,9 +531,7 @@ class WorkflowCycleManage(WorkflowIterationCycleManage):
                 total_tokens=self._task_state.total_tokens,
                 total_steps=self._task_state.total_steps,
                 status=WorkflowRunStatus.FAILED,
-                error=event.error,
-                conversation_id=conversation_id,
-                trace_manager=trace_manager
+                error=event.error
             )
         else:
             if self._task_state.latest_node_execution_info:
@@ -584,9 +546,7 @@ class WorkflowCycleManage(WorkflowIterationCycleManage):
                 start_at=self._task_state.start_at,
                 total_tokens=self._task_state.total_tokens,
                 total_steps=self._task_state.total_steps,
-                outputs=outputs,
-                conversation_id=conversation_id,
-                trace_manager=trace_manager
+                outputs=outputs
             )
 
         self._task_state.workflow_run_id = workflow_run.id