upgrade version

.devcontainer/post_start_command.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-cd api && poetry install
+poetry install -C api

.github/actions/setup-poetry/action.yml

@@ -1,36 +0,0 @@
-name: Setup Poetry and Python
-
-inputs:
-  python-version:
-    description: Python version to use and the Poetry installed with
-    required: true
-    default: '3.10'
-  poetry-version:
-    description: Poetry version to set up
-    required: true
-    default: '1.8.4'
-  poetry-lockfile:
-    description: Path to the Poetry lockfile to restore cache from
-    required: true
-    default: ''
-
-runs:
-  using: composite
-  steps:
-    - name: Set up Python ${{ inputs.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ inputs.python-version }}
-        cache: pip
-
-    - name: Install Poetry
-      shell: bash
-      run: pip install poetry==${{ inputs.poetry-version }}
-
-    - name: Restore Poetry cache
-      if: ${{ inputs.poetry-lockfile != '' }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ inputs.python-version }}
-        cache: poetry
-        cache-dependency-path: ${{ inputs.poetry-lockfile }}

.github/workflows/api-tests.yml

@@ -7,7 +7,6 @@ on:
     paths:
       - api/**
       - docker/**
-      - .github/workflows/api-tests.yml
 
 concurrency:
   group: api-tests-${{ github.head_ref || github.run_id }}
@@ -28,11 +27,16 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Setup Poetry and Python ${{ matrix.python-version }}
-        uses: ./.github/actions/setup-poetry
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          poetry-lockfile: api/poetry.lock
+          cache-dependency-path: |
+            api/pyproject.toml
+            api/poetry.lock
+
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v3
 
       - name: Check Poetry lockfile
         run: |
@@ -63,7 +67,7 @@ jobs:
         run: sh .github/workflows/expose_service_ports.sh
 
       - name: Set up Sandbox
-        uses: hoverkraft-tech/compose-action@v2.0.2
+        uses: hoverkraft-tech/compose-action@v2.0.0
         with:
           compose-file: |
             docker/docker-compose.middleware.yaml
@@ -73,3 +77,21 @@ jobs:
 
       - name: Run Workflow
         run: poetry run -C api bash dev/pytest/pytest_workflow.sh
+
+      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch)
+        uses: hoverkraft-tech/compose-action@v2.0.0
+        with:
+          compose-file: |
+            docker/docker-compose.yaml
+          services: |
+            weaviate
+            qdrant
+            etcd
+            minio
+            milvus-standalone
+            pgvecto-rs
+            pgvector
+            chroma
+            elasticsearch
+      - name: Test Vector Stores
+        run: poetry run -C api bash dev/pytest/pytest_vdb.sh

.github/workflows/build-push.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - "main"
       - "deploy/dev"
+      - "feat/upgrade-unstructured-version-10.2"
   release:
     types: [published]
 
@@ -49,7 +50,7 @@ jobs:
           echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           username: ${{ env.DOCKERHUB_USER }}
           password: ${{ env.DOCKERHUB_TOKEN }}
@@ -114,7 +115,7 @@ jobs:
           merge-multiple: true
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           username: ${{ env.DOCKERHUB_USER }}
           password: ${{ env.DOCKERHUB_TOKEN }}

.github/workflows/db-migration-test.yml

@@ -6,7 +6,6 @@ on:
       - main
     paths:
       - api/migrations/**
-      - .github/workflows/db-migration-test.yml
 
 concurrency:
   group: db-migration-test-${{ github.ref }}
@@ -15,15 +14,25 @@ concurrency:
 jobs:
   db-migration-test:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.10"
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Setup Poetry and Python
-        uses: ./.github/actions/setup-poetry
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
         with:
-          poetry-lockfile: api/poetry.lock
+          python-version: ${{ matrix.python-version }}
+          cache-dependency-path: |
+            api/pyproject.toml
+            api/poetry.lock
+
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v3
 
       - name: Install dependencies
         run: poetry install -C api
@@ -34,7 +43,7 @@ jobs:
           cp middleware.env.example middleware.env
 
       - name: Set up Middlewares
-        uses: hoverkraft-tech/compose-action@v2.0.2
+        uses: hoverkraft-tech/compose-action@v2.0.0
         with:
           compose-file: |
             docker/docker-compose.middleware.yaml

.github/workflows/expose_service_ports.sh

@@ -7,7 +7,5 @@ yq eval '.services["milvus-standalone"].ports += ["19530:19530"]' -i docker/dock
 yq eval '.services.pgvector.ports += ["5433:5432"]' -i docker/docker-compose.yaml
 yq eval '.services["pgvecto-rs"].ports += ["5431:5432"]' -i docker/docker-compose.yaml
 yq eval '.services["elasticsearch"].ports += ["9200:9200"]' -i docker/docker-compose.yaml
-yq eval '.services.couchbase-server.ports += ["8091-8096:8091-8096"]' -i docker/docker-compose.yaml
-yq eval '.services.couchbase-server.ports += ["11210:11210"]' -i docker/docker-compose.yaml
 
-echo "Ports exposed for sandbox, weaviate, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch, couchbase"
+echo "Ports exposed for sandbox, weaviate, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch"

.github/workflows/style.yml

@@ -22,28 +22,34 @@ jobs:
         id: changed-files
         uses: tj-actions/changed-files@v45
         with:
-          files: |
-            api/**
-            .github/workflows/style.yml
+          files: api/**
 
-      - name: Setup Poetry and Python
+      - name: Set up Python
+        uses: actions/setup-python@v5
         if: steps.changed-files.outputs.any_changed == 'true'
-        uses: ./.github/actions/setup-poetry
+        with:
+          python-version: '3.10'
 
-      - name: Install dependencies
+      - name: Install Poetry
+        if: steps.changed-files.outputs.any_changed == 'true'
+        uses: abatilo/actions-poetry@v3
+
+      - name: Python dependencies
         if: steps.changed-files.outputs.any_changed == 'true'
         run: poetry install -C api --only lint
 
       - name: Ruff check
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: |
-          poetry run -C api ruff check ./api
-          poetry run -C api ruff format --check ./api
+        run: poetry run -C api ruff check ./api
 
       - name: Dotenv check
         if: steps.changed-files.outputs.any_changed == 'true'
         run: poetry run -C api dotenv-linter ./api/.env.example ./web/.env.example
 
+      - name: Ruff formatter check
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: poetry run -C api ruff format --check ./api
+
       - name: Lint hints
         if: failure()
         run: echo "Please run 'dev/reformat' to fix the fixable linting errors."

.github/workflows/vdb-tests.yml

@@ -1,71 +0,0 @@
-name: Run VDB Tests
-
-on:
-  pull_request:
-    branches:
-      - main
-    paths:
-      - api/core/rag/datasource/**
-      - docker/**
-      - .github/workflows/vdb-tests.yml
-
-concurrency:
-  group: vdb-tests-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
-jobs:
-  test:
-    name: VDB Tests
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version:
-          - "3.10"
-          - "3.11"
-          - "3.12"
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Setup Poetry and Python ${{ matrix.python-version }}
-        uses: ./.github/actions/setup-poetry
-        with:
-          python-version: ${{ matrix.python-version }}
-          poetry-lockfile: api/poetry.lock
-
-      - name: Check Poetry lockfile
-        run: |
-          poetry check -C api --lock
-          poetry show -C api
-
-      - name: Install dependencies
-        run: poetry install -C api --with dev
-
-      - name: Set up dotenvs
-        run: |
-          cp docker/.env.example docker/.env
-          cp docker/middleware.env.example docker/middleware.env
-
-      - name: Expose Service Ports
-        run: sh .github/workflows/expose_service_ports.sh
-
-      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch, Couchbase)
-        uses: hoverkraft-tech/compose-action@v2.0.2
-        with:
-          compose-file: |
-            docker/docker-compose.yaml
-          services: |
-            weaviate
-            qdrant
-            couchbase-server
-            etcd
-            minio
-            milvus-standalone
-            pgvecto-rs
-            pgvector
-            chroma
-            elasticsearch
-
-      - name: Test Vector Stores
-        run: poetry run -C api bash dev/pytest/pytest_vdb.sh

.gitignore

@@ -173,9 +173,6 @@ docker/volumes/myscale/log/*
 docker/volumes/unstructured/*
 docker/volumes/pgvector/data/*
 docker/volumes/pgvecto_rs/data/*
-docker/volumes/couchbase/*
-docker/volumes/oceanbase/*
-!docker/volumes/oceanbase/init.d
 
 docker/nginx/conf.d/default.conf
 docker/nginx/ssl/*
@@ -192,4 +189,4 @@ pyrightconfig.json
 api/.vscode
 
 .idea/
-.vscode
+.vscode

CONTRIBUTING.md

@@ -81,7 +81,7 @@ Dify requires the following dependencies to build, make sure they're installed o
 
 Dify is composed of a backend and a frontend. Navigate to the backend directory by `cd api/`, then follow the [Backend README](api/README.md) to install it. In a separate terminal, navigate to the frontend directory by `cd web/`, then follow the [Frontend README](web/README.md) to install.
 
-Check the [installation FAQ](https://docs.dify.ai/learn-more/faq/install-faq) for a list of common issues and steps to troubleshoot.
+Check the [installation FAQ](https://docs.dify.ai/learn-more/faq/self-host-faq) for a list of common issues and steps to troubleshoot.
 
 ### 5. Visit dify in your browser
 

CONTRIBUTING_VI.md

@@ -79,7 +79,7 @@ Dify yêu cầu các phụ thuộc sau để build, hãy đảm bảo chúng đ
 
 Dify bao gồm một backend và một frontend. Đi đến thư mục backend bằng lệnh `cd api/`, sau đó làm theo hướng dẫn trong [README của Backend](api/README.md) để cài đặt. Trong một terminal khác, đi đến thư mục frontend bằng lệnh `cd web/`, sau đó làm theo hướng dẫn trong [README của Frontend](web/README.md) để cài đặt.
 
-Kiểm tra [FAQ về cài đặt](https://docs.dify.ai/learn-more/faq/install-faq) để xem danh sách các vấn đề thường gặp và các bước khắc phục.
+Kiểm tra [FAQ về cài đặt](https://docs.dify.ai/learn-more/faq/self-host-faq) để xem danh sách các vấn đề thường gặp và các bước khắc phục.
 
 ### 5. Truy cập Dify trong trình duyệt của bạn
 

README.md

@@ -46,33 +46,9 @@
 </p>
 
 
-Dify is an open-source LLM app development platform. Its intuitive interface combines agentic AI workflow, RAG pipeline, agent capabilities, model management, observability features and more, letting you quickly go from prototype to production. 
+Dify is an open-source LLM app development platform. Its intuitive interface combines AI workflow, RAG pipeline, agent capabilities, model management, observability features and more, letting you quickly go from prototype to production. Here's a list of the core features:
+</br> </br>
 
-## Quick start
-> Before installing Dify, make sure your machine meets the following minimum system requirements:
-> 
->- CPU >= 2 Core
->- RAM >= 4 GiB
-
-</br>
-
-The easiest way to start the Dify server is through [docker compose](docker/docker-compose.yaml). Before running Dify with the following commands, make sure that [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) are installed on your machine:
-
-```bash
-cd dify
-cd docker
-cp .env.example .env
-docker compose up -d
-```
-
-After running, you can access the Dify dashboard in your browser at [http://localhost/install](http://localhost/install) and start the initialization process.
-
-#### Seeking help
-Please refer to our [FAQ](https://docs.dify.ai/getting-started/install-self-hosted/faqs) if you encounter problems setting up Dify. Reach out to [the community and us](#community--contact) if you are still having issues.
-
-> If you'd like to contribute to Dify or do additional development, refer to our [guide to deploying from source code](https://docs.dify.ai/getting-started/install-self-hosted/local-source-code)
-
-## Key features
 **1. Workflow**: 
   Build and test powerful AI workflows on a visual canvas, leveraging all the following features and beyond.
 
@@ -103,6 +79,73 @@ Please refer to our [FAQ](https://docs.dify.ai/getting-started/install-self-host
   All of Dify's offerings come with corresponding APIs, so you could effortlessly integrate Dify into your own business logic.
 
 
+## Feature comparison
+<table style="width: 100%;">
+  <tr>
+    <th align="center">Feature</th>
+    <th align="center">Dify.AI</th>
+    <th align="center">LangChain</th>
+    <th align="center">Flowise</th>
+    <th align="center">OpenAI Assistants API</th>
+  </tr>
+  <tr>
+    <td align="center">Programming Approach</td>
+    <td align="center">API + App-oriented</td>
+    <td align="center">Python Code</td>
+    <td align="center">App-oriented</td>
+    <td align="center">API-oriented</td>
+  </tr>
+  <tr>
+    <td align="center">Supported LLMs</td>
+    <td align="center">Rich Variety</td>
+    <td align="center">Rich Variety</td>
+    <td align="center">Rich Variety</td>
+    <td align="center">OpenAI-only</td>
+  </tr>
+  <tr>
+    <td align="center">RAG Engine</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+  </tr>
+  <tr>
+    <td align="center">Agent</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">✅</td>
+  </tr>
+  <tr>
+    <td align="center">Workflow</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+  </tr>
+  <tr>
+    <td align="center">Observability</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">❌</td>
+  </tr>
+  <tr>
+    <td align="center">Enterprise Features (SSO/Access control)</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">❌</td>
+    <td align="center">❌</td>
+  </tr>
+  <tr>
+    <td align="center">Local Deployment</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+  </tr>
+</table>
+
 ## Using Dify
 
 - **Cloud </br>**
@@ -124,7 +167,28 @@ Star Dify on GitHub and be instantly notified of new releases.
 ![star-us](https://github.com/langgenius/dify/assets/13230914/b823edc1-6388-4e25-ad45-2f6b187adbb4)
 
 
-## Advanced Setup
+
+## Quick start
+> Before installing Dify, make sure your machine meets the following minimum system requirements:
+> 
+>- CPU >= 2 Core
+>- RAM >= 4 GiB
+
+</br>
+
+The easiest way to start the Dify server is to run our [docker-compose.yml](docker/docker-compose.yaml) file. Before running the installation command, make sure that [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) are installed on your machine:
+
+```bash
+cd docker
+cp .env.example .env
+docker compose up -d
+```
+
+After running, you can access the Dify dashboard in your browser at [http://localhost/install](http://localhost/install) and start the initialization process.
+
+> If you'd like to contribute to Dify or do additional development, refer to our [guide to deploying from source code](https://docs.dify.ai/getting-started/install-self-hosted/local-source-code)
+
+## Next steps
 
 If you need to customize the configuration, please refer to the comments in our [.env.example](docker/.env.example) file and update the corresponding values in your `.env` file. Additionally, you might need to make adjustments to the `docker-compose.yaml` file itself, such as changing image versions, port mappings, or volume mounts, based on your specific deployment environment and requirements. After making any changes, please re-run `docker-compose up -d`. You can find the full list of available environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
@@ -152,6 +216,12 @@ At the same time, please consider supporting Dify by sharing it on social media
 
 > We are looking for contributors to help with translating Dify to languages other than Mandarin or English. If you are interested in helping, please see the [i18n README](https://github.com/langgenius/dify/blob/main/web/i18n/README.md) for more information, and leave us a comment in the `global-users` channel of our [Discord Community Server](https://discord.gg/8Tpq4AcN9c).
 
+**Contributors**
+
+<a href="https://github.com/langgenius/dify/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=langgenius/dify" />
+</a>
+
 ## Community & contact
 
 * [Github Discussion](https://github.com/langgenius/dify/discussions). Best for: sharing feedback and asking questions.
@@ -159,12 +229,6 @@ At the same time, please consider supporting Dify by sharing it on social media
 * [Discord](https://discord.gg/FngNHpbcY7). Best for: sharing your applications and hanging out with the community.
 * [X(Twitter)](https://twitter.com/dify_ai). Best for: sharing your applications and hanging out with the community.
 
-**Contributors**
-
-<a href="https://github.com/langgenius/dify/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=langgenius/dify" />
-</a>
-
 ## Star history
 
 [![Star History Chart](https://api.star-history.com/svg?repos=langgenius/dify&type=Date)](https://star-history.com/#langgenius/dify&Date)
@@ -177,4 +241,3 @@ To protect your privacy, please avoid posting security issues on GitHub. Instead
 ## License
 
 This repository is available under the [Dify Open Source License](LICENSE), which is essentially Apache 2.0 with a few additional restrictions.
-

README_PT.md

@@ -1,241 +0,0 @@
-![cover-v5-optimized](https://github.com/langgenius/dify/assets/13230914/f9e19af5-61ba-4119-b926-d10c4c06ebab)
-
-<p align="center">
-  📌 <a href="https://dify.ai/blog/introducing-dify-workflow-file-upload-a-demo-on-ai-podcast">Introduzindo o Dify Workflow com Upload de Arquivo: Recrie o Podcast Google NotebookLM</a>
-</p>
-
-<p align="center">
-  <a href="https://cloud.dify.ai">Dify Cloud</a> ·
-  <a href="https://docs.dify.ai/getting-started/install-self-hosted">Auto-hospedagem</a> ·
-  <a href="https://docs.dify.ai">Documentação</a> ·
-  <a href="https://udify.app/chat/22L1zSxg6yW1cWQg">Consultas empresariais</a>
-</p>
-
-<p align="center">
-    <a href="https://dify.ai" target="_blank">
-        <img alt="Static Badge" src="https://img.shields.io/badge/Product-F04438"></a>
-    <a href="https://dify.ai/pricing" target="_blank">
-        <img alt="Static Badge" src="https://img.shields.io/badge/free-pricing?logo=free&color=%20%23155EEF&label=pricing&labelColor=%20%23528bff"></a>
-    <a href="https://discord.gg/FngNHpbcY7" target="_blank">
-        <img src="https://img.shields.io/discord/1082486657678311454?logo=discord&labelColor=%20%235462eb&logoColor=%20%23f5f5f5&color=%20%235462eb"
-            alt="chat on Discord"></a>
-    <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
-        <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
-            alt="follow on X(Twitter)"></a>
-    <a href="https://hub.docker.com/u/langgenius" target="_blank">
-        <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
-    <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
-        <img alt="Commits last month" src="https://img.shields.io/github/commit-activity/m/langgenius/dify?labelColor=%20%2332b583&color=%20%2312b76a"></a>
-    <a href="https://github.com/langgenius/dify/" target="_blank">
-        <img alt="Issues closed" src="https://img.shields.io/github/issues-search?query=repo%3Alanggenius%2Fdify%20is%3Aclosed&label=issues%20closed&labelColor=%20%237d89b0&color=%20%235d6b98"></a>
-    <a href="https://github.com/langgenius/dify/discussions/" target="_blank">
-        <img alt="Discussion posts" src="https://img.shields.io/github/discussions/langgenius/dify?labelColor=%20%239b8afb&color=%20%237a5af8"></a>
-</p>
-
-<p align="center">
-  <a href="./README.md"><img alt="README em Inglês" src="https://img.shields.io/badge/English-d9d9d9"></a>
-  <a href="./README_CN.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-d9d9d9"></a>
-  <a href="./README_JA.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-d9d9d9"></a>
-  <a href="./README_ES.md"><img alt="README em Espanhol" src="https://img.shields.io/badge/Español-d9d9d9"></a>
-  <a href="./README_FR.md"><img alt="README em Francês" src="https://img.shields.io/badge/Français-d9d9d9"></a>
-  <a href="./README_KL.md"><img alt="README tlhIngan Hol" src="https://img.shields.io/badge/Klingon-d9d9d9"></a>
-  <a href="./README_KR.md"><img alt="README em Coreano" src="https://img.shields.io/badge/한국어-d9d9d9"></a>
-  <a href="./README_AR.md"><img alt="README em Árabe" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
-  <a href="./README_TR.md"><img alt="README em Turco" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
-  <a href="./README_VI.md"><img alt="README em Vietnamita" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
-  <a href="./README_PT.md"><img alt="README em Português - BR" src="https://img.shields.io/badge/Portugu%C3%AAs-BR?style=flat&label=BR&color=d9d9d9"></a>
-</p>
-
-Dify é uma plataforma de desenvolvimento de aplicativos LLM de código aberto. Sua interface intuitiva combina workflow de IA, pipeline RAG, capacidades de agente, gerenciamento de modelos, recursos de observabilidade e muito mais, permitindo que você vá rapidamente do protótipo à produção. Aqui está uma lista das principais funcionalidades:
-</br> </br>
-
-**1. Workflow**: 
-  Construa e teste workflows poderosos de IA em uma interface visual, aproveitando todos os recursos a seguir e muito mais.
-
-
-  https://github.com/langgenius/dify/assets/13230914/356df23e-1604-483d-80a6-9517ece318aa
-
-
-
-**2. Suporte abrangente a modelos**: 
-  Integração perfeita com centenas de LLMs proprietários e de código aberto de diversas provedoras e soluções auto-hospedadas, abrangendo GPT, Mistral, Llama3 e qualquer modelo compatível com a API da OpenAI. A lista completa de provedores suportados pode ser encontrada [aqui](https://docs.dify.ai/getting-started/readme/model-providers).
-
-![providers-v5](https://github.com/langgenius/dify/assets/13230914/5a17bdbe-097a-4100-8363-40255b70f6e3)
-
-
-**3. IDE de Prompt**: 
-  Interface intuitiva para criação de prompts, comparação de desempenho de modelos e adição de recursos como conversão de texto para fala em um aplicativo baseado em chat. 
-
-**4. Pipeline RAG**: 
-  Extensas capacidades de RAG que cobrem desde a ingestão de documentos até a recuperação, com suporte nativo para extração de texto de PDFs, PPTs e outros formatos de documentos comuns.
-
-**5. Capacidades de agente**: 
-  Você pode definir agentes com base em LLM Function Calling ou ReAct e adicionar ferramentas pré-construídas ou personalizadas para o agente. O Dify oferece mais de 50 ferramentas integradas para agentes de IA, como Google Search, DALL·E, Stable Diffusion e WolframAlpha.
-
-**6. LLMOps**: 
-  Monitore e analise os registros e o desempenho do aplicativo ao longo do tempo. É possível melhorar continuamente prompts, conjuntos de dados e modelos com base nos dados de produção e anotações.
-
-**7. Backend como Serviço**: 
-  Todas os recursos do Dify vêm com APIs correspondentes, permitindo que você integre o Dify sem esforço na lógica de negócios da sua empresa.
-
-
-## Comparação de recursos
-<table style="width: 100%;">
-  <tr>
-    <th align="center">Recurso</th>
-    <th align="center">Dify.AI</th>
-    <th align="center">LangChain</th>
-    <th align="center">Flowise</th>
-    <th align="center">OpenAI Assistants API</th>
-  </tr>
-  <tr>
-    <td align="center">Abordagem de Programação</td>
-    <td align="center">Orientada a API + Aplicativo</td>
-    <td align="center">Código Python</td>
-    <td align="center">Orientada a Aplicativo</td>
-    <td align="center">Orientada a API</td>
-  </tr>
-  <tr>
-    <td align="center">LLMs Suportados</td>
-    <td align="center">Variedade Rica</td>
-    <td align="center">Variedade Rica</td>
-    <td align="center">Variedade Rica</td>
-    <td align="center">Apenas OpenAI</td>
-  </tr>
-  <tr>
-    <td align="center">RAG Engine</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-  </tr>
-  <tr>
-    <td align="center">Agente</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">✅</td>
-  </tr>
-  <tr>
-    <td align="center">Workflow</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-  </tr>
-  <tr>
-    <td align="center">Observabilidade</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">❌</td>
-  </tr>
-  <tr>
-    <td align="center">Recursos Empresariais (SSO/Controle de Acesso)</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">❌</td>
-    <td align="center">❌</td>
-  </tr>
-  <tr>
-    <td align="center">Implantação Local</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-  </tr>
-</table>
-
-## Usando o Dify
-
-- **Nuvem </br>**
-Oferecemos o serviço [Dify Cloud](https://dify.ai) para qualquer pessoa experimentar sem nenhuma configuração. Ele fornece todas as funcionalidades da versão auto-hospedada, incluindo 200 chamadas GPT-4 gratuitas no plano sandbox.
-
-- **Auto-hospedagem do Dify Community Edition</br>**
-Configure rapidamente o Dify no seu ambiente com este [guia inicial](#quick-start).
-Use nossa [documentação](https://docs.dify.ai) para referências adicionais e instruções mais detalhadas.
-
-- **Dify para empresas/organizações</br>**
-Oferecemos recursos adicionais voltados para empresas. [Envie suas perguntas através deste chatbot](https://udify.app/chat/22L1zSxg6yW1cWQg) ou [envie-nos um e-mail](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry) para discutir necessidades empresariais. </br>
-  > Para startups e pequenas empresas que utilizam AWS, confira o [Dify Premium no AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6) e implemente no seu próprio AWS VPC com um clique. É uma oferta AMI acessível com a opção de criar aplicativos com logotipo e marca personalizados.
-
-
-## Mantendo-se atualizado
-
-Dê uma estrela no Dify no GitHub e seja notificado imediatamente sobre novos lançamentos.
-
-![star-us](https://github.com/langgenius/dify/assets/13230914/b823edc1-6388-4e25-ad45-2f6b187adbb4)
-
-
-
-## Início rápido
-> Antes de instalar o Dify, certifique-se de que sua máquina atenda aos seguintes requisitos mínimos de sistema:
-> 
->- CPU >= 2 Núcleos
->- RAM >= 4 GiB
-
-</br>
-
-A maneira mais fácil de iniciar o servidor Dify é executar nosso arquivo [docker-compose.yml](docker/docker-compose.yaml). Antes de rodar o comando de instalação, certifique-se de que o [Docker](https://docs.docker.com/get-docker/) e o [Docker Compose](https://docs.docker.com/compose/install/) estão instalados na sua máquina:
-
-```bash
-cd docker
-cp .env.example .env
-docker compose up -d
-```
-
-Após a execução, você pode acessar o painel do Dify no navegador em [http://localhost/install](http://localhost/install) e iniciar o processo de inicialização.
-
-> Se você deseja contribuir com o Dify ou fazer desenvolvimento adicional, consulte nosso [guia para implantar a partir do código fonte](https://docs.dify.ai/getting-started/install-self-hosted/local-source-code).
-
-## Próximos passos
-
-Se precisar personalizar a configuração, consulte os comentários no nosso arquivo [.env.example](docker/.env.example) e atualize os valores correspondentes no seu arquivo `.env`. Além disso, talvez seja necessário fazer ajustes no próprio arquivo `docker-compose.yaml`, como alterar versões de imagem, mapeamentos de portas ou montagens de volumes, com base no seu ambiente de implantação específico e nas suas necessidades. Após fazer quaisquer alterações, execute novamente `docker-compose up -d`. Você pode encontrar a lista completa de variáveis de ambiente disponíveis [aqui](https://docs.dify.ai/getting-started/install-self-hosted/environments).
-
-Se deseja configurar uma instalação de alta disponibilidade, há [Helm Charts](https://helm.sh/) e arquivos YAML contribuídos pela comunidade que permitem a implantação do Dify no Kubernetes.
-
-- [Helm Chart de @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
-- [Helm Chart de @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
-- [Arquivo YAML de @Winson-030](https://github.com/Winson-030/dify-kubernetes)
-
-#### Usando o Terraform para Implantação
-
-Implante o Dify na Plataforma Cloud com um único clique usando [terraform](https://www.terraform.io/)
-
-##### Azure Global
-- [Azure Terraform por @nikawang](https://github.com/nikawang/dify-azure-terraform)
-
-##### Google Cloud
-- [Google Cloud Terraform por @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
-
-## Contribuindo
-
-Para aqueles que desejam contribuir com código, veja nosso [Guia de Contribuição](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). 
-Ao mesmo tempo, considere apoiar o Dify compartilhando-o nas redes sociais e em eventos e conferências.
-
-> Estamos buscando contribuidores para ajudar na tradução do Dify para idiomas além de Mandarim e Inglês. Se você tiver interesse em ajudar, consulte o [README i18n](https://github.com/langgenius/dify/blob/main/web/i18n/README.md) para mais informações e deixe-nos um comentário no canal `global-users` em nosso [Servidor da Comunidade no Discord](https://discord.gg/8Tpq4AcN9c).
-
-**Contribuidores**
-
-<a href="https://github.com/langgenius/dify/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=langgenius/dify" />
-</a>
-
-## Comunidade e contato
-
-* [Discussões no GitHub](https://github.com/langgenius/dify/discussions). Melhor para: compartilhar feedback e fazer perguntas.
-* [Problemas no GitHub](https://github.com/langgenius/dify/issues). Melhor para: relatar bugs encontrados no Dify.AI e propor novos recursos. Veja nosso [Guia de Contribuição](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
-* [Discord](https://discord.gg/FngNHpbcY7). Melhor para: compartilhar suas aplicações e interagir com a comunidade.
-* [X(Twitter)](https://twitter.com/dify_ai). Melhor para: compartilhar suas aplicações e interagir com a comunidade.
-
-## Histórico de estrelas
-
-[![Gráfico de Histórico de Estrelas](https://api.star-history.com/svg?repos=langgenius/dify&type=Date)](https://star-history.com/#langgenius/dify&Date)
-
-## Divulgação de segurança
-
-Para proteger sua privacidade, evite postar problemas de segurança no GitHub. Em vez disso, envie suas perguntas para security@dify.ai e forneceremos uma resposta mais detalhada.
-
-## Licença
-
-Este repositório está disponível sob a [Licença de Código Aberto Dify](LICENSE), que é essencialmente Apache 2.0 com algumas restrições adicionais.

api/.env.example

@@ -120,8 +120,7 @@ SUPABASE_URL=your-server-url
 WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 
-
-# Vector database configuration, support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector, couchbase, vikingdb, upstash, lindorm, oceanbase
+# Vector database configuration, support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector, vikingdb, upstash
 VECTOR_STORE=weaviate
 
 # Weaviate configuration
@@ -137,13 +136,6 @@ QDRANT_CLIENT_TIMEOUT=20
 QDRANT_GRPC_ENABLED=false
 QDRANT_GRPC_PORT=6334
 
-#Couchbase configuration
-COUCHBASE_CONNECTION_STRING=127.0.0.1
-COUCHBASE_USER=Administrator
-COUCHBASE_PASSWORD=password
-COUCHBASE_BUCKET_NAME=Embeddings
-COUCHBASE_SCOPE_NAME=_default
-
 # Milvus configuration
 MILVUS_URI=http://127.0.0.1:19530
 MILVUS_TOKEN=
@@ -203,20 +195,6 @@ TIDB_VECTOR_USER=xxx.root
 TIDB_VECTOR_PASSWORD=xxxxxx
 TIDB_VECTOR_DATABASE=dify
 
-# Tidb on qdrant configuration
-TIDB_ON_QDRANT_URL=http://127.0.0.1
-TIDB_ON_QDRANT_API_KEY=dify
-TIDB_ON_QDRANT_CLIENT_TIMEOUT=20
-TIDB_ON_QDRANT_GRPC_ENABLED=false
-TIDB_ON_QDRANT_GRPC_PORT=6334
-TIDB_PUBLIC_KEY=dify
-TIDB_PRIVATE_KEY=dify
-TIDB_API_URL=http://127.0.0.1
-TIDB_IAM_API_URL=http://127.0.0.1
-TIDB_REGION=regions/aws-us-east-1
-TIDB_PROJECT_ID=dify
-TIDB_SPEND_LIMIT=100
-
 # Chroma configuration
 CHROMA_HOST=127.0.0.1
 CHROMA_PORT=8000
@@ -264,20 +242,6 @@ VIKINGDB_SCHEMA=http
 VIKINGDB_CONNECTION_TIMEOUT=30
 VIKINGDB_SOCKET_TIMEOUT=30
 
-# Lindorm configuration
-LINDORM_URL=http://ld-*******************-proxy-search-pub.lindorm.aliyuncs.com:30070
-LINDORM_USERNAME=admin
-LINDORM_PASSWORD=admin
-
-# OceanBase Vector configuration
-OCEANBASE_VECTOR_HOST=127.0.0.1
-OCEANBASE_VECTOR_PORT=2881
-OCEANBASE_VECTOR_USER=root@test
-OCEANBASE_VECTOR_PASSWORD=difyai123456
-OCEANBASE_VECTOR_DATABASE=test
-OCEANBASE_MEMORY_LIMIT=6G
-
-
 # Upload configuration
 UPLOAD_FILE_SIZE_LIMIT=15
 UPLOAD_FILE_BATCH_LIMIT=5
@@ -285,9 +249,8 @@ UPLOAD_IMAGE_FILE_SIZE_LIMIT=10
 UPLOAD_VIDEO_FILE_SIZE_LIMIT=100
 UPLOAD_AUDIO_FILE_SIZE_LIMIT=50
 
-# Model configuration
+# Model Configuration
 MULTIMODAL_SEND_IMAGE_FORMAT=base64
-MULTIMODAL_SEND_VIDEO_FORMAT=base64
 PROMPT_GENERATION_MAX_TOKENS=512
 CODE_GENERATION_MAX_TOKENS=1024
 
@@ -321,21 +284,13 @@ ETL_TYPE=dify
 UNSTRUCTURED_API_URL=
 UNSTRUCTURED_API_KEY=
 
-#ssrf
 SSRF_PROXY_HTTP_URL=
 SSRF_PROXY_HTTPS_URL=
 SSRF_DEFAULT_MAX_RETRIES=3
-SSRF_DEFAULT_TIME_OUT=5
-SSRF_DEFAULT_CONNECT_TIME_OUT=5
-SSRF_DEFAULT_READ_TIME_OUT=5
-SSRF_DEFAULT_WRITE_TIME_OUT=5
 
 BATCH_UPLOAD_LIMIT=10
 KEYWORD_DATA_SOURCE_TYPE=database
 
-# Workflow file upload limit
-WORKFLOW_FILE_UPLOAD_LIMIT=10
-
 # CODE EXECUTION CONFIGURATION
 CODE_EXECUTION_ENDPOINT=http://127.0.0.1:8194
 CODE_EXECUTION_API_KEY=dify-sandbox
@@ -367,10 +322,6 @@ LOG_FILE=
 LOG_FILE_MAX_SIZE=20
 # Log file max backup count
 LOG_FILE_BACKUP_COUNT=5
-# Log dateformat
-LOG_DATEFORMAT=%Y-%m-%d %H:%M:%S
-# Log Timezone
-LOG_TZ=UTC
 
 # Indexing configuration
 INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=1000
@@ -400,5 +351,3 @@ POSITION_PROVIDER_EXCLUDES=
 
 # Reset password token expiry minutes
 RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5
-
-CREATE_TIDB_SERVICE_JOB_ENABLED=false

api/Dockerfile

@@ -4,7 +4,7 @@ FROM python:3.10-slim-bookworm AS base
 WORKDIR /app/api
 
 # Install Poetry
-ENV POETRY_VERSION=1.8.4
+ENV POETRY_VERSION=1.8.3
 
 # if you located in China, you can use aliyun mirror to speed up
 # RUN pip install --no-cache-dir poetry==${POETRY_VERSION} -i https://mirrors.aliyun.com/pypi/simple/
@@ -49,25 +49,25 @@ ENV TZ=UTC
 WORKDIR /app/api
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends curl nodejs libgmp-dev libmpfr-dev libmpc-dev \
-    # if you located in China, you can use aliyun mirror to speed up
-    # && echo "deb http://mirrors.aliyun.com/debian testing main" > /etc/apt/sources.list \
-    && echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \
-    && apt-get update \
-    # For Security
-    && apt-get install -y --no-install-recommends expat=2.6.3-2 libldap-2.5-0=2.5.18+dfsg-3+b1 perl=5.40.0-7 libsqlite3-0=3.46.1-1 zlib1g=1:1.3.dfsg+really1.3.1-1+b1 \
-    # install a chinese font to support the use of tools like matplotlib
-    && apt-get install -y fonts-noto-cjk \
+    # Install dependencies
+    && apt-get install -y --no-install-recommends \
+        # basic environment
+        curl nodejs libgmp-dev libmpfr-dev libmpc-dev \
+        # For Security
+        expat libldap-2.5-0 perl libsqlite3-0 zlib1g \
+        # install a chinese font to support the use of tools like matplotlib
+        fonts-noto-cjk \
+        # install libmagic to support the use of python-magic guess MIMETYPE
+        libmagic1 \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
-
 # Copy Python environment and packages
 ENV VIRTUAL_ENV=/app/api/.venv
 COPY --from=packages ${VIRTUAL_ENV} ${VIRTUAL_ENV}
 ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 
 # Download nltk data
-RUN python -c "import nltk; nltk.download('punkt'); nltk.download('averaged_perceptron_tagger')"
+RUN python -c "import nltk; nltk.download('punkt'); nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger')"
 
 # Copy source code
 COPY . /app/api/

api/README.md

@@ -76,13 +76,13 @@
 1. Install dependencies for both the backend and the test environment
 
    ```bash
-   poetry install -C api --with dev
+   poetry install --with dev
    ```
 
 2. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml`
 
    ```bash
+   cd ../
    poetry run -C api bash dev/pytest/pytest_all_tests.sh
    ```
 
-

api/app.py

@@ -1,9 +1,8 @@
 import os
-import sys
 
 from configs import dify_config
 
-if not dify_config.DEBUG:
+if os.environ.get("DEBUG", "false").lower() != "true":
     from gevent import monkey
 
     monkey.patch_all()
@@ -30,9 +29,6 @@ from models import account, dataset, model, source, task, tool, tools, web  # no
 
 # DO NOT REMOVE ABOVE
 
-if sys.version_info[:2] == (3, 10):
-    print("Warning: Python 3.10 will not be supported in the next version.")
-
 
 warnings.simplefilter("ignore", ResourceWarning)
 

api/app_factory.py

@@ -1,8 +1,6 @@
 import os
 
-from configs import dify_config
-
-if not dify_config.DEBUG:
+if os.environ.get("DEBUG", "false").lower() != "true":
     from gevent import monkey
 
     monkey.patch_all()

api/commands.py

@@ -278,8 +278,6 @@ def migrate_knowledge_vector_database():
         VectorType.BAIDU,
         VectorType.VIKINGDB,
         VectorType.UPSTASH,
-        VectorType.COUCHBASE,
-        VectorType.OCEANBASE,
     }
     page = 1
     while True:

api/configs/feature/__init__.py

@@ -109,7 +109,7 @@ class CodeExecutionSandboxConfig(BaseSettings):
     )
 
     CODE_MAX_PRECISION: PositiveInt = Field(
-        description="Maximum number of decimal places for floating-point numbers in code execution",
+        description="mMaximum number of decimal places for floating-point numbers in code execution",
         default=20,
     )
 
@@ -216,11 +216,6 @@ class FileUploadConfig(BaseSettings):
         default=20,
     )
 
-    WORKFLOW_FILE_UPLOAD_LIMIT: PositiveInt = Field(
-        description="Maximum number of files allowed in a workflow upload operation",
-        default=10,
-    )
-
 
 class HttpConfig(BaseSettings):
     """
@@ -276,16 +271,6 @@ class HttpConfig(BaseSettings):
         default=1 * 1024 * 1024,
     )
 
-    SSRF_DEFAULT_MAX_RETRIES: PositiveInt = Field(
-        description="Maximum number of retries for network requests (SSRF)",
-        default=3,
-    )
-
-    SSRF_PROXY_ALL_URL: Optional[str] = Field(
-        description="Proxy URL for HTTP or HTTPS requests to prevent Server-Side Request Forgery (SSRF)",
-        default=None,
-    )
-
     SSRF_PROXY_HTTP_URL: Optional[str] = Field(
         description="Proxy URL for HTTP requests to prevent Server-Side Request Forgery (SSRF)",
         default=None,
@@ -296,26 +281,6 @@ class HttpConfig(BaseSettings):
         default=None,
     )
 
-    SSRF_DEFAULT_TIME_OUT: PositiveFloat = Field(
-        description="The default timeout period used for network requests (SSRF)",
-        default=5,
-    )
-
-    SSRF_DEFAULT_CONNECT_TIME_OUT: PositiveFloat = Field(
-        description="The default connect timeout period used for network requests (SSRF)",
-        default=5,
-    )
-
-    SSRF_DEFAULT_READ_TIME_OUT: PositiveFloat = Field(
-        description="The default read timeout period used for network requests (SSRF)",
-        default=5,
-    )
-
-    SSRF_DEFAULT_WRITE_TIME_OUT: PositiveFloat = Field(
-        description="The default write timeout period used for network requests (SSRF)",
-        default=5,
-    )
-
     RESPECT_XFORWARD_HEADERS_ENABLED: bool = Field(
         description="Enable or disable the X-Forwarded-For Proxy Fix middleware from Werkzeug"
         " to respect X-* headers to redirect clients",
@@ -376,7 +341,7 @@ class LoggingConfig(BaseSettings):
 
     LOG_TZ: Optional[str] = Field(
         description="Timezone for log timestamps (e.g., 'America/New_York')",
-        default="UTC",
+        default=None,
     )
 
 
@@ -611,11 +576,6 @@ class DataSetConfig(BaseSettings):
         default=500,
     )
 
-    CREATE_TIDB_SERVICE_JOB_ENABLED: bool = Field(
-        description="Enable or disable create tidb service job",
-        default=False,
-    )
-
 
 class WorkspaceConfig(BaseSettings):
     """
@@ -639,17 +599,12 @@ class IndexingConfig(BaseSettings):
     )
 
 
-class VisionFormatConfig(BaseSettings):
+class ImageFormatConfig(BaseSettings):
     MULTIMODAL_SEND_IMAGE_FORMAT: Literal["base64", "url"] = Field(
         description="Format for sending images in multimodal contexts ('base64' or 'url'), default is base64",
         default="base64",
     )
 
-    MULTIMODAL_SEND_VIDEO_FORMAT: Literal["base64", "url"] = Field(
-        description="Format for sending videos in multimodal contexts ('base64' or 'url'), default is base64",
-        default="base64",
-    )
-
 
 class CeleryBeatConfig(BaseSettings):
     CELERY_BEAT_SCHEDULER_TIME: int = Field(
@@ -752,7 +707,7 @@ class FeatureConfig(
     FileAccessConfig,
     FileUploadConfig,
     HttpConfig,
-    VisionFormatConfig,
+    ImageFormatConfig,
     InnerAPIConfig,
     IndexingConfig,
     LoggingConfig,

api/configs/middleware/__init__.py

@@ -16,14 +16,10 @@ from configs.middleware.storage.supabase_storage_config import SupabaseStorageCo
 from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
 from configs.middleware.storage.volcengine_tos_storage_config import VolcengineTOSStorageConfig
 from configs.middleware.vdb.analyticdb_config import AnalyticdbConfig
-from configs.middleware.vdb.baidu_vector_config import BaiduVectorDBConfig
 from configs.middleware.vdb.chroma_config import ChromaConfig
-from configs.middleware.vdb.couchbase_config import CouchbaseConfig
 from configs.middleware.vdb.elasticsearch_config import ElasticsearchConfig
-from configs.middleware.vdb.lindorm_config import LindormConfig
 from configs.middleware.vdb.milvus_config import MilvusConfig
 from configs.middleware.vdb.myscale_config import MyScaleConfig
-from configs.middleware.vdb.oceanbase_config import OceanBaseVectorConfig
 from configs.middleware.vdb.opensearch_config import OpenSearchConfig
 from configs.middleware.vdb.oracle_config import OracleConfig
 from configs.middleware.vdb.pgvector_config import PGVectorConfig
@@ -255,13 +251,9 @@ class MiddlewareConfig(
     TiDBVectorConfig,
     WeaviateConfig,
     ElasticsearchConfig,
-    CouchbaseConfig,
     InternalTestConfig,
     VikingDBConfig,
     UpstashConfig,
     TidbOnQdrantConfig,
-    LindormConfig,
-    OceanBaseVectorConfig,
-    BaiduVectorDBConfig,
 ):
     pass

api/configs/middleware/vdb/couchbase_config.py

@@ -1,34 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field
-
-
-class CouchbaseConfig(BaseModel):
-    """
-    Couchbase configs
-    """
-
-    COUCHBASE_CONNECTION_STRING: Optional[str] = Field(
-        description="COUCHBASE connection string",
-        default=None,
-    )
-
-    COUCHBASE_USER: Optional[str] = Field(
-        description="COUCHBASE user",
-        default=None,
-    )
-
-    COUCHBASE_PASSWORD: Optional[str] = Field(
-        description="COUCHBASE password",
-        default=None,
-    )
-
-    COUCHBASE_BUCKET_NAME: Optional[str] = Field(
-        description="COUCHBASE bucket name",
-        default=None,
-    )
-
-    COUCHBASE_SCOPE_NAME: Optional[str] = Field(
-        description="COUCHBASE scope name",
-        default=None,
-    )

api/configs/middleware/vdb/lindorm_config.py

@@ -1,23 +0,0 @@
-from typing import Optional
-
-from pydantic import Field
-from pydantic_settings import BaseSettings
-
-
-class LindormConfig(BaseSettings):
-    """
-    Lindorm configs
-    """
-
-    LINDORM_URL: Optional[str] = Field(
-        description="Lindorm url",
-        default=None,
-    )
-    LINDORM_USERNAME: Optional[str] = Field(
-        description="Lindorm user",
-        default=None,
-    )
-    LINDORM_PASSWORD: Optional[str] = Field(
-        description="Lindorm password",
-        default=None,
-    )

api/configs/middleware/vdb/oceanbase_config.py

@@ -1,35 +0,0 @@
-from typing import Optional
-
-from pydantic import Field, PositiveInt
-from pydantic_settings import BaseSettings
-
-
-class OceanBaseVectorConfig(BaseSettings):
-    """
-    Configuration settings for OceanBase Vector database
-    """
-
-    OCEANBASE_VECTOR_HOST: Optional[str] = Field(
-        description="Hostname or IP address of the OceanBase Vector server (e.g. 'localhost')",
-        default=None,
-    )
-
-    OCEANBASE_VECTOR_PORT: Optional[PositiveInt] = Field(
-        description="Port number on which the OceanBase Vector server is listening (default is 2881)",
-        default=2881,
-    )
-
-    OCEANBASE_VECTOR_USER: Optional[str] = Field(
-        description="Username for authenticating with the OceanBase Vector database",
-        default=None,
-    )
-
-    OCEANBASE_VECTOR_PASSWORD: Optional[str] = Field(
-        description="Password for authenticating with the OceanBase Vector database",
-        default=None,
-    )
-
-    OCEANBASE_VECTOR_DATABASE: Optional[str] = Field(
-        description="Name of the OceanBase Vector database to connect to",
-        default=None,
-    )

api/configs/middleware/vdb/tidb_on_qdrant_config.py

@@ -63,8 +63,3 @@ class TidbOnQdrantConfig(BaseSettings):
         description="Tidb project id",
         default=None,
     )
-
-    TIDB_SPEND_LIMIT: Optional[int] = Field(
-        description="Tidb spend limit",
-        default=100,
-    )

api/configs/packaging/__init__.py

@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
 
     CURRENT_VERSION: str = Field(
         description="Dify version",
-        default="0.11.1",
+        default="0.10.2",
     )
 
     COMMIT_SHA: str = Field(

api/controllers/common/errors.py

@@ -1,6 +0,0 @@
-from werkzeug.exceptions import HTTPException
-
-
-class FilenameNotExistsError(HTTPException):
-    code = 400
-    description = "The specified filename does not exist."

api/controllers/common/fields.py

@@ -1,24 +0,0 @@
-from flask_restful import fields
-
-parameters__system_parameters = {
-    "image_file_size_limit": fields.Integer,
-    "video_file_size_limit": fields.Integer,
-    "audio_file_size_limit": fields.Integer,
-    "file_size_limit": fields.Integer,
-    "workflow_file_upload_limit": fields.Integer,
-}
-
-parameters_fields = {
-    "opening_statement": fields.String,
-    "suggested_questions": fields.Raw,
-    "suggested_questions_after_answer": fields.Raw,
-    "speech_to_text": fields.Raw,
-    "text_to_speech": fields.Raw,
-    "retriever_resource": fields.Raw,
-    "annotation_reply": fields.Raw,
-    "more_like_this": fields.Raw,
-    "user_input_form": fields.Raw,
-    "sensitive_word_avoidance": fields.Raw,
-    "file_upload": fields.Raw,
-    "system_parameters": fields.Nested(parameters__system_parameters),
-}

api/controllers/common/helpers.py

@@ -1,97 +0,0 @@
-import mimetypes
-import os
-import re
-import urllib.parse
-from collections.abc import Mapping
-from typing import Any
-from uuid import uuid4
-
-import httpx
-from pydantic import BaseModel
-
-from configs import dify_config
-
-
-class FileInfo(BaseModel):
-    filename: str
-    extension: str
-    mimetype: str
-    size: int
-
-
-def guess_file_info_from_response(response: httpx.Response):
-    url = str(response.url)
-    # Try to extract filename from URL
-    parsed_url = urllib.parse.urlparse(url)
-    url_path = parsed_url.path
-    filename = os.path.basename(url_path)
-
-    # If filename couldn't be extracted, use Content-Disposition header
-    if not filename:
-        content_disposition = response.headers.get("Content-Disposition")
-        if content_disposition:
-            filename_match = re.search(r'filename="?(.+)"?', content_disposition)
-            if filename_match:
-                filename = filename_match.group(1)
-
-    # If still no filename, generate a unique one
-    if not filename:
-        unique_name = str(uuid4())
-        filename = f"{unique_name}"
-
-    # Guess MIME type from filename first, then URL
-    mimetype, _ = mimetypes.guess_type(filename)
-    if mimetype is None:
-        mimetype, _ = mimetypes.guess_type(url)
-    if mimetype is None:
-        # If guessing fails, use Content-Type from response headers
-        mimetype = response.headers.get("Content-Type", "application/octet-stream")
-
-    extension = os.path.splitext(filename)[1]
-
-    # Ensure filename has an extension
-    if not extension:
-        extension = mimetypes.guess_extension(mimetype) or ".bin"
-        filename = f"{filename}{extension}"
-
-    return FileInfo(
-        filename=filename,
-        extension=extension,
-        mimetype=mimetype,
-        size=int(response.headers.get("Content-Length", -1)),
-    )
-
-
-def get_parameters_from_feature_dict(*, features_dict: Mapping[str, Any], user_input_form: list[dict[str, Any]]):
-    return {
-        "opening_statement": features_dict.get("opening_statement"),
-        "suggested_questions": features_dict.get("suggested_questions", []),
-        "suggested_questions_after_answer": features_dict.get("suggested_questions_after_answer", {"enabled": False}),
-        "speech_to_text": features_dict.get("speech_to_text", {"enabled": False}),
-        "text_to_speech": features_dict.get("text_to_speech", {"enabled": False}),
-        "retriever_resource": features_dict.get("retriever_resource", {"enabled": False}),
-        "annotation_reply": features_dict.get("annotation_reply", {"enabled": False}),
-        "more_like_this": features_dict.get("more_like_this", {"enabled": False}),
-        "user_input_form": user_input_form,
-        "sensitive_word_avoidance": features_dict.get(
-            "sensitive_word_avoidance", {"enabled": False, "type": "", "configs": []}
-        ),
-        "file_upload": features_dict.get(
-            "file_upload",
-            {
-                "image": {
-                    "enabled": False,
-                    "number_limits": 3,
-                    "detail": "high",
-                    "transfer_methods": ["remote_url", "local_file"],
-                }
-            },
-        ),
-        "system_parameters": {
-            "image_file_size_limit": dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT,
-            "video_file_size_limit": dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT,
-            "audio_file_size_limit": dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT,
-            "file_size_limit": dify_config.UPLOAD_FILE_SIZE_LIMIT,
-            "workflow_file_upload_limit": dify_config.WORKFLOW_FILE_UPLOAD_LIMIT,
-        },
-    }

api/controllers/console/__init__.py

@@ -2,21 +2,9 @@ from flask import Blueprint
 
 from libs.external_api import ExternalApi
 
-from .files import FileApi, FilePreviewApi, FileSupportTypeApi
-from .remote_files import RemoteFileInfoApi, RemoteFileUploadApi
-
 bp = Blueprint("console", __name__, url_prefix="/console/api")
 api = ExternalApi(bp)
 
-# File
-api.add_resource(FileApi, "/files/upload")
-api.add_resource(FilePreviewApi, "/files/<uuid:file_id>/preview")
-api.add_resource(FileSupportTypeApi, "/files/support-type")
-
-# Remote files
-api.add_resource(RemoteFileInfoApi, "/remote-files/<path:url>")
-api.add_resource(RemoteFileUploadApi, "/remote-files/upload")
-
 # Import other controllers
 from . import admin, apikey, extension, feature, ping, setup, version
 
@@ -55,6 +43,7 @@ from .datasets import (
     datasets_document,
     datasets_segments,
     external,
+    file,
     hit_testing,
     website,
 )

api/controllers/console/apikey.py

@@ -10,7 +10,8 @@ from models.dataset import Dataset
 from models.model import ApiToken, App
 
 from . import api
-from .wraps import account_initialization_required, setup_required
+from .setup import setup_required
+from .wraps import account_initialization_required
 
 api_key_fields = {
     "id": fields.String,

api/controllers/console/app/advanced_prompt_template.py

@@ -1,7 +1,8 @@
 from flask_restful import Resource, reqparse
 
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from libs.login import login_required
 from services.advanced_prompt_template_service import AdvancedPromptTemplateService
 

api/controllers/console/app/agent.py

@@ -2,7 +2,8 @@ from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from libs.helper import uuid_value
 from libs.login import login_required
 from models.model import AppMode

api/controllers/console/app/annotation.py

@@ -6,11 +6,8 @@ from werkzeug.exceptions import Forbidden
 from controllers.console import api
 from controllers.console.app.error import NoFileUploadedError
 from controllers.console.datasets.error import TooManyFilesError
-from controllers.console.wraps import (
-    account_initialization_required,
-    cloud_edition_billing_resource_check,
-    setup_required,
-)
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_redis import redis_client
 from fields.annotation_fields import (
     annotation_fields,

api/controllers/console/app/app.py

@@ -6,11 +6,8 @@ from werkzeug.exceptions import BadRequest, Forbidden, abort
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import (
-    account_initialization_required,
-    cloud_edition_billing_resource_check,
-    setup_required,
-)
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from core.ops.ops_trace_manager import OpsTraceManager
 from fields.app_fields import (
     app_detail_fields,

api/controllers/console/app/audio.py

@@ -18,7 +18,8 @@ from controllers.console.app.error import (
     UnsupportedAudioTypeError,
 )
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
 from libs.login import login_required

api/controllers/console/app/completion.py

@@ -15,7 +15,8 @@ from controllers.console.app.error import (
     ProviderQuotaExceededError,
 )
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom

api/controllers/console/app/conversation.py

@@ -10,7 +10,8 @@ from werkzeug.exceptions import Forbidden, NotFound
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.app.entities.app_invoke_entities import InvokeFrom
 from extensions.ext_database import db
 from fields.conversation_fields import (

api/controllers/console/app/conversation_variables.py

@@ -4,7 +4,8 @@ from sqlalchemy.orm import Session
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
 from fields.conversation_variable_fields import paginated_conversation_variable_fields
 from libs.login import login_required

api/controllers/console/app/generator.py

@@ -10,7 +10,8 @@ from controllers.console.app.error import (
     ProviderNotInitializeError,
     ProviderQuotaExceededError,
 )
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.llm_generator.llm_generator import LLMGenerator
 from core.model_runtime.errors.invoke import InvokeError

api/controllers/console/app/message.py

@@ -14,11 +14,8 @@ from controllers.console.app.error import (
 )
 from controllers.console.app.wraps import get_app_model
 from controllers.console.explore.error import AppSuggestedQuestionsAfterAnswerDisabledError
-from controllers.console.wraps import (
-    account_initialization_required,
-    cloud_edition_billing_resource_check,
-    setup_required,
-)
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from core.app.entities.app_invoke_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError

api/controllers/console/app/model_config.py

@@ -6,7 +6,8 @@ from flask_restful import Resource
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.agent.entities import AgentToolEntity
 from core.tools.tool_manager import ToolManager
 from core.tools.utils.configuration import ToolParameterConfigurationManager

api/controllers/console/app/ops_trace.py

@@ -2,7 +2,8 @@ from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.app.error import TracingConfigCheckError, TracingConfigIsExist, TracingConfigNotExist
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from libs.login import login_required
 from services.ops_service import OpsService
 

api/controllers/console/app/site.py

@@ -7,7 +7,8 @@ from werkzeug.exceptions import Forbidden, NotFound
 from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
 from fields.app_fields import app_site_fields
 from libs.login import login_required

api/controllers/console/app/statistic.py

@@ -8,7 +8,8 @@ from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
 from libs.helper import DatetimeString
 from libs.login import login_required

api/controllers/console/app/workflow.py

@@ -9,7 +9,8 @@ import services
 from controllers.console import api
 from controllers.console.app.error import ConversationCompletedError, DraftWorkflowNotExist, DraftWorkflowNotSync
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom
 from factories import variable_factory

api/controllers/console/app/workflow_app_log.py

@@ -3,7 +3,8 @@ from flask_restful.inputs import int_range
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from fields.workflow_app_log_fields import workflow_app_log_pagination_fields
 from libs.login import login_required
 from models import App

api/controllers/console/app/workflow_run.py

@@ -3,7 +3,8 @@ from flask_restful.inputs import int_range
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from fields.workflow_run_fields import (
     advanced_chat_workflow_run_pagination_fields,
     workflow_run_detail_fields,

api/controllers/console/app/workflow_statistic.py

@@ -8,7 +8,8 @@ from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
 from libs.helper import DatetimeString
 from libs.login import login_required

api/controllers/console/auth/data_source_bearer_auth.py

@@ -7,7 +7,8 @@ from controllers.console.auth.error import ApiKeyAuthFailedError
 from libs.login import login_required
 from services.auth.api_key_auth_service import ApiKeyAuthService
 
-from ..wraps import account_initialization_required, setup_required
+from ..setup import setup_required
+from ..wraps import account_initialization_required
 
 
 class ApiKeyAuthDataSource(Resource):

api/controllers/console/auth/data_source_oauth.py

@@ -11,7 +11,8 @@ from controllers.console import api
 from libs.login import login_required
 from libs.oauth_data_source import NotionOAuth
 
-from ..wraps import account_initialization_required, setup_required
+from ..setup import setup_required
+from ..wraps import account_initialization_required
 
 
 def get_oauth_providers():

api/controllers/console/auth/forgot_password.py

@@ -13,7 +13,7 @@ from controllers.console.auth.error import (
     PasswordMismatchError,
 )
 from controllers.console.error import EmailSendIpLimitError, NotAllowedRegister
-from controllers.console.wraps import setup_required
+from controllers.console.setup import setup_required
 from events.tenant_event import tenant_was_created
 from extensions.ext_database import db
 from libs.helper import email, extract_remote_ip

api/controllers/console/auth/login.py

@@ -20,7 +20,7 @@ from controllers.console.error import (
     NotAllowedCreateWorkspace,
     NotAllowedRegister,
 )
-from controllers.console.wraps import setup_required
+from controllers.console.setup import setup_required
 from events.tenant_event import tenant_was_created
 from libs.helper import email, extract_remote_ip
 from libs.password import valid_password

api/controllers/console/billing/billing.py

@@ -2,7 +2,8 @@ from flask_login import current_user
 from flask_restful import Resource, reqparse
 
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, only_edition_cloud, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, only_edition_cloud
 from libs.login import login_required
 from services.billing_service import BillingService
 

api/controllers/console/datasets/data_source.py

@@ -7,7 +7,8 @@ from flask_restful import Resource, marshal_with, reqparse
 from werkzeug.exceptions import NotFound
 
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.indexing_runner import IndexingRunner
 from core.rag.extractor.entity.extract_setting import ExtractSetting
 from core.rag.extractor.notion_extractor import NotionExtractor

api/controllers/console/datasets/datasets.py

@@ -10,7 +10,8 @@ from controllers.console import api
 from controllers.console.apikey import api_key_fields, api_key_list
 from controllers.console.app.error import ProviderNotInitializeError
 from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.indexing_runner import IndexingRunner
 from core.model_runtime.entities.model_entities import ModelType
@@ -456,7 +457,7 @@ class DatasetIndexingEstimateApi(Resource):
             )
         except LLMBadRequestError:
             raise ProviderNotInitializeError(
-                "No Embedding Model available. Please configure a valid provider " "in the Settings -> Model Provider."
+                "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
@@ -620,7 +621,6 @@ class DatasetRetrievalSettingApi(Resource):
             case (
                 VectorType.MILVUS
                 | VectorType.RELYT
-                | VectorType.PGVECTOR
                 | VectorType.TIDB_VECTOR
                 | VectorType.CHROMA
                 | VectorType.TENCENT
@@ -628,7 +628,6 @@ class DatasetRetrievalSettingApi(Resource):
                 | VectorType.BAIDU
                 | VectorType.VIKINGDB
                 | VectorType.UPSTASH
-                | VectorType.OCEANBASE
             ):
                 return {"retrieval_method": [RetrievalMethod.SEMANTIC_SEARCH.value]}
             case (
@@ -641,8 +640,6 @@ class DatasetRetrievalSettingApi(Resource):
                 | VectorType.ELASTICSEARCH
                 | VectorType.PGVECTOR
                 | VectorType.TIDB_ON_QDRANT
-                | VectorType.LINDORM
-                | VectorType.COUCHBASE
             ):
                 return {
                     "retrieval_method": [
@@ -671,7 +668,6 @@ class DatasetRetrievalSettingMockApi(Resource):
                 | VectorType.BAIDU
                 | VectorType.VIKINGDB
                 | VectorType.UPSTASH
-                | VectorType.OCEANBASE
             ):
                 return {"retrieval_method": [RetrievalMethod.SEMANTIC_SEARCH.value]}
             case (
@@ -682,9 +678,7 @@ class DatasetRetrievalSettingMockApi(Resource):
                 | VectorType.MYSCALE
                 | VectorType.ORACLE
                 | VectorType.ELASTICSEARCH
-                | VectorType.COUCHBASE
                 | VectorType.PGVECTOR
-                | VectorType.LINDORM
             ):
                 return {
                     "retrieval_method": [

api/controllers/console/datasets/datasets_document.py

@@ -24,11 +24,8 @@ from controllers.console.datasets.error import (
     InvalidActionError,
     InvalidMetadataError,
 )
-from controllers.console.wraps import (
-    account_initialization_required,
-    cloud_edition_billing_resource_check,
-    setup_required,
-)
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from core.errors.error import (
     LLMBadRequestError,
     ModelCurrentlyNotSupportError,
@@ -317,11 +314,8 @@ class DatasetInitApi(Resource):
                 raise ValueError("embedding model and embedding model provider are required for high quality indexing.")
             try:
                 model_manager = ModelManager()
-                model_manager.get_model_instance(
-                    tenant_id=current_user.current_tenant_id,
-                    provider=args["embedding_model_provider"],
-                    model_type=ModelType.TEXT_EMBEDDING,
-                    model=args["embedding_model"],
+                model_manager.get_default_model_instance(
+                    tenant_id=current_user.current_tenant_id, model_type=ModelType.TEXT_EMBEDDING
                 )
             except InvokeAuthorizationError:
                 raise ProviderNotInitializeError(
@@ -948,7 +942,7 @@ class DocumentRetryApi(DocumentResource):
                     raise DocumentAlreadyFinishedError()
                 retry_documents.append(document)
             except Exception as e:
-                logging.exception(f"Document {document_id} retry failed: {str(e)}")
+                logging.error(f"Document {document_id} retry failed: {str(e)}")
                 continue
         # retry document
         DocumentService.retry_document(dataset_id, retry_documents)

api/controllers/console/datasets/datasets_segments.py

@@ -11,11 +11,11 @@ import services
 from controllers.console import api
 from controllers.console.app.error import ProviderNotInitializeError
 from controllers.console.datasets.error import InvalidActionError, NoFileUploadedError, TooManyFilesError
+from controllers.console.setup import setup_required
 from controllers.console.wraps import (
     account_initialization_required,
     cloud_edition_billing_knowledge_limit_check,
     cloud_edition_billing_resource_check,
-    setup_required,
 )
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.model_manager import ModelManager

api/controllers/console/datasets/external.py

@@ -6,7 +6,8 @@ from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 import services
 from controllers.console import api
 from controllers.console.datasets.error import DatasetNameDuplicateError
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from fields.dataset_fields import dataset_detail_fields
 from libs.login import login_required
 from services.dataset_service import DatasetService

api/controllers/console/datasets/file.py

@@ -1,26 +1,25 @@
+import urllib.parse
+
 from flask import request
 from flask_login import current_user
-from flask_restful import Resource, marshal_with
+from flask_restful import Resource, marshal_with, reqparse
 
 import services
 from configs import dify_config
 from constants import DOCUMENT_EXTENSIONS
-from controllers.common.errors import FilenameNotExistsError
-from controllers.console.wraps import (
-    account_initialization_required,
-    cloud_edition_billing_resource_check,
-    setup_required,
-)
-from fields.file_fields import file_fields, upload_config_fields
-from libs.login import login_required
-from services.file_service import FileService
-
-from .error import (
+from controllers.console import api
+from controllers.console.datasets.error import (
     FileTooLargeError,
     NoFileUploadedError,
     TooManyFilesError,
     UnsupportedFileTypeError,
 )
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
+from core.helper import ssrf_proxy
+from fields.file_fields import file_fields, remote_file_info_fields, upload_config_fields
+from libs.login import login_required
+from services.file_service import FileService
 
 PREVIEW_WORDS_LIMIT = 3000
 
@@ -37,7 +36,6 @@ class FileApi(Resource):
             "image_file_size_limit": dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT,
             "video_file_size_limit": dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT,
             "audio_file_size_limit": dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT,
-            "workflow_file_upload_limit": dify_config.WORKFLOW_FILE_UPLOAD_LIMIT,
         }, 200
 
     @setup_required
@@ -46,29 +44,21 @@ class FileApi(Resource):
     @marshal_with(file_fields)
     @cloud_edition_billing_resource_check("documents")
     def post(self):
+        # get file from request
         file = request.files["file"]
-        source = request.form.get("source")
 
+        parser = reqparse.RequestParser()
+        parser.add_argument("source", type=str, required=False, location="args")
+        source = parser.parse_args().get("source")
+
+        # check file
         if "file" not in request.files:
             raise NoFileUploadedError()
 
         if len(request.files) > 1:
             raise TooManyFilesError()
-
-        if not file.filename:
-            raise FilenameNotExistsError
-
-        if source not in ("datasets", None):
-            source = None
-
         try:
-            upload_file = FileService.upload_file(
-                filename=file.filename,
-                content=file.read(),
-                mimetype=file.mimetype,
-                user=current_user,
-                source=source,
-            )
+            upload_file = FileService.upload_file(file=file, user=current_user, source=source)
         except services.errors.file.FileTooLargeError as file_too_large_error:
             raise FileTooLargeError(file_too_large_error.description)
         except services.errors.file.UnsupportedFileTypeError:
@@ -93,3 +83,23 @@ class FileSupportTypeApi(Resource):
     @account_initialization_required
     def get(self):
         return {"allowed_extensions": DOCUMENT_EXTENSIONS}
+
+
+class RemoteFileInfoApi(Resource):
+    @marshal_with(remote_file_info_fields)
+    def get(self, url):
+        decoded_url = urllib.parse.unquote(url)
+        try:
+            response = ssrf_proxy.head(decoded_url)
+            return {
+                "file_type": response.headers.get("Content-Type", "application/octet-stream"),
+                "file_length": int(response.headers.get("Content-Length", 0)),
+            }
+        except Exception as e:
+            return {"error": str(e)}, 400
+
+
+api.add_resource(FileApi, "/files/upload")
+api.add_resource(FilePreviewApi, "/files/<uuid:file_id>/preview")
+api.add_resource(FileSupportTypeApi, "/files/support-type")
+api.add_resource(RemoteFileInfoApi, "/remote-files/<path:url>")

api/controllers/console/datasets/hit_testing.py

@@ -2,7 +2,8 @@ from flask_restful import Resource
 
 from controllers.console import api
 from controllers.console.datasets.hit_testing_base import DatasetsHitTestingBase
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from libs.login import login_required
 
 

api/controllers/console/datasets/website.py

@@ -2,7 +2,8 @@ from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.datasets.error import WebsiteCrawlError
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from libs.login import login_required
 from services.website_service import WebsiteService
 

api/controllers/console/error.py

@@ -62,27 +62,3 @@ class EmailSendIpLimitError(BaseHTTPException):
     error_code = "email_send_ip_limit"
     description = "Too many emails have been sent from this IP address recently. Please try again later."
     code = 429
-
-
-class FileTooLargeError(BaseHTTPException):
-    error_code = "file_too_large"
-    description = "File size exceeded. {message}"
-    code = 413
-
-
-class UnsupportedFileTypeError(BaseHTTPException):
-    error_code = "unsupported_file_type"
-    description = "File type not allowed."
-    code = 415
-
-
-class TooManyFilesError(BaseHTTPException):
-    error_code = "too_many_files"
-    description = "Only one file is allowed."
-    code = 400
-
-
-class NoFileUploadedError(BaseHTTPException):
-    error_code = "no_file_uploaded"
-    description = "Please upload your file."
-    code = 400

api/controllers/console/explore/parameter.py

@@ -1,7 +1,6 @@
-from flask_restful import marshal_with
+from flask_restful import fields, marshal_with
 
-from controllers.common import fields
-from controllers.common import helpers as controller_helpers
+from configs import dify_config
 from controllers.console import api
 from controllers.console.app.error import AppUnavailableError
 from controllers.console.explore.wraps import InstalledAppResource
@@ -12,14 +11,43 @@ from services.app_service import AppService
 class AppParameterApi(InstalledAppResource):
     """Resource for app variables."""
 
-    @marshal_with(fields.parameters_fields)
+    variable_fields = {
+        "key": fields.String,
+        "name": fields.String,
+        "description": fields.String,
+        "type": fields.String,
+        "default": fields.String,
+        "max_length": fields.Integer,
+        "options": fields.List(fields.String),
+    }
+
+    system_parameters_fields = {
+        "image_file_size_limit": fields.Integer,
+        "video_file_size_limit": fields.Integer,
+        "audio_file_size_limit": fields.Integer,
+        "file_size_limit": fields.Integer,
+    }
+
+    parameters_fields = {
+        "opening_statement": fields.String,
+        "suggested_questions": fields.Raw,
+        "suggested_questions_after_answer": fields.Raw,
+        "speech_to_text": fields.Raw,
+        "text_to_speech": fields.Raw,
+        "retriever_resource": fields.Raw,
+        "annotation_reply": fields.Raw,
+        "more_like_this": fields.Raw,
+        "user_input_form": fields.Raw,
+        "sensitive_word_avoidance": fields.Raw,
+        "file_upload": fields.Raw,
+        "system_parameters": fields.Nested(system_parameters_fields),
+    }
+
+    @marshal_with(parameters_fields)
     def get(self, installed_app: InstalledApp):
         """Retrieve app parameters."""
         app_model = installed_app.app
 
-        if app_model is None:
-            raise AppUnavailableError()
-
         if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}:
             workflow = app_model.workflow
             if workflow is None:
@@ -29,16 +57,43 @@ class AppParameterApi(InstalledAppResource):
             user_input_form = workflow.user_input_form(to_old_structure=True)
         else:
             app_model_config = app_model.app_model_config
-            if app_model_config is None:
-                raise AppUnavailableError()
-
             features_dict = app_model_config.to_dict()
 
             user_input_form = features_dict.get("user_input_form", [])
 
-        return controller_helpers.get_parameters_from_feature_dict(
-            features_dict=features_dict, user_input_form=user_input_form
-        )
+        return {
+            "opening_statement": features_dict.get("opening_statement"),
+            "suggested_questions": features_dict.get("suggested_questions", []),
+            "suggested_questions_after_answer": features_dict.get(
+                "suggested_questions_after_answer", {"enabled": False}
+            ),
+            "speech_to_text": features_dict.get("speech_to_text", {"enabled": False}),
+            "text_to_speech": features_dict.get("text_to_speech", {"enabled": False}),
+            "retriever_resource": features_dict.get("retriever_resource", {"enabled": False}),
+            "annotation_reply": features_dict.get("annotation_reply", {"enabled": False}),
+            "more_like_this": features_dict.get("more_like_this", {"enabled": False}),
+            "user_input_form": user_input_form,
+            "sensitive_word_avoidance": features_dict.get(
+                "sensitive_word_avoidance", {"enabled": False, "type": "", "configs": []}
+            ),
+            "file_upload": features_dict.get(
+                "file_upload",
+                {
+                    "image": {
+                        "enabled": False,
+                        "number_limits": 3,
+                        "detail": "high",
+                        "transfer_methods": ["remote_url", "local_file"],
+                    }
+                },
+            ),
+            "system_parameters": {
+                "image_file_size_limit": dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT,
+                "video_file_size_limit": dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT,
+                "audio_file_size_limit": dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT,
+                "file_size_limit": dify_config.UPLOAD_FILE_SIZE_LIMIT,
+            },
+        }
 
 
 class ExploreAppMetaApi(InstalledAppResource):

api/controllers/console/extension.py

@@ -3,7 +3,8 @@ from flask_restful import Resource, marshal_with, reqparse
 
 from constants import HIDDEN_VALUE
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from fields.api_based_extension_fields import api_based_extension_fields
 from libs.login import login_required
 from models.api_based_extension import APIBasedExtension

api/controllers/console/feature.py

@@ -5,7 +5,8 @@ from libs.login import login_required
 from services.feature_service import FeatureService
 
 from . import api
-from .wraps import account_initialization_required, cloud_utm_record, setup_required
+from .setup import setup_required
+from .wraps import account_initialization_required, cloud_utm_record
 
 
 class FeatureApi(Resource):

api/controllers/console/remote_files.py

@@ -1,81 +0,0 @@
-import urllib.parse
-from typing import cast
-
-import httpx
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
-
-import services
-from controllers.common import helpers
-from core.file import helpers as file_helpers
-from core.helper import ssrf_proxy
-from fields.file_fields import file_fields_with_signed_url, remote_file_info_fields
-from models.account import Account
-from services.file_service import FileService
-
-from .error import (
-    FileTooLargeError,
-    UnsupportedFileTypeError,
-)
-
-
-class RemoteFileInfoApi(Resource):
-    @marshal_with(remote_file_info_fields)
-    def get(self, url):
-        decoded_url = urllib.parse.unquote(url)
-        resp = ssrf_proxy.head(decoded_url)
-        if resp.status_code != httpx.codes.OK:
-            # failed back to get method
-            resp = ssrf_proxy.get(decoded_url, timeout=3)
-        resp.raise_for_status()
-        return {
-            "file_type": resp.headers.get("Content-Type", "application/octet-stream"),
-            "file_length": int(resp.headers.get("Content-Length", 0)),
-        }
-
-
-class RemoteFileUploadApi(Resource):
-    @marshal_with(file_fields_with_signed_url)
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("url", type=str, required=True, help="URL is required")
-        args = parser.parse_args()
-
-        url = args["url"]
-
-        resp = ssrf_proxy.head(url=url)
-        if resp.status_code != httpx.codes.OK:
-            resp = ssrf_proxy.get(url=url, timeout=3)
-        resp.raise_for_status()
-
-        file_info = helpers.guess_file_info_from_response(resp)
-
-        if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):
-            raise FileTooLargeError
-
-        content = resp.content if resp.request.method == "GET" else ssrf_proxy.get(url).content
-
-        try:
-            user = cast(Account, current_user)
-            upload_file = FileService.upload_file(
-                filename=file_info.filename,
-                content=content,
-                mimetype=file_info.mimetype,
-                user=user,
-                source_url=url,
-            )
-        except services.errors.file.FileTooLargeError as file_too_large_error:
-            raise FileTooLargeError(file_too_large_error.description)
-        except services.errors.file.UnsupportedFileTypeError:
-            raise UnsupportedFileTypeError()
-
-        return {
-            "id": upload_file.id,
-            "name": upload_file.name,
-            "size": upload_file.size,
-            "extension": upload_file.extension,
-            "url": file_helpers.get_signed_file_url(upload_file_id=upload_file.id),
-            "mime_type": upload_file.mime_type,
-            "created_by": upload_file.created_by,
-            "created_at": upload_file.created_at,
-        }, 201

api/controllers/console/setup.py

@@ -1,3 +1,5 @@
+from functools import wraps
+
 from flask import request
 from flask_restful import Resource, reqparse
 
@@ -8,7 +10,7 @@ from models.model import DifySetup
 from services.account_service import RegisterService, TenantService
 
 from . import api
-from .error import AlreadySetupError, NotInitValidateError
+from .error import AlreadySetupError, NotInitValidateError, NotSetupError
 from .init_validate import get_init_validate_status
 from .wraps import only_edition_self_hosted
 
@@ -50,10 +52,26 @@ class SetupApi(Resource):
         return {"result": "success"}, 201
 
 
+def setup_required(view):
+    @wraps(view)
+    def decorated(*args, **kwargs):
+        # check setup
+        if not get_init_validate_status():
+            raise NotInitValidateError()
+
+        elif not get_setup_status():
+            raise NotSetupError()
+
+        return view(*args, **kwargs)
+
+    return decorated
+
+
 def get_setup_status():
     if dify_config.EDITION == "SELF_HOSTED":
         return DifySetup.query.first()
-    return True
+    else:
+        return True
 
 
 api.add_resource(SetupApi, "/setup")

api/controllers/console/tag/tags.py

@@ -4,7 +4,8 @@ from flask_restful import Resource, marshal_with, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from fields.tag_fields import tag_fields
 from libs.login import login_required
 from models.model import Tag

api/controllers/console/version.py

@@ -3,7 +3,6 @@ import logging
 
 import requests
 from flask_restful import Resource, reqparse
-from packaging import version
 
 from configs import dify_config
 
@@ -48,15 +47,43 @@ class VersionApi(Resource):
 
 
 def _has_new_version(*, latest_version: str, current_version: str) -> bool:
-    try:
-        latest = version.parse(latest_version)
-        current = version.parse(current_version)
+    def parse_version(version: str) -> tuple:
+        # Split version into parts and pre-release suffix if any
+        parts = version.split("-")
+        version_parts = parts[0].split(".")
+        pre_release = parts[1] if len(parts) > 1 else None
 
-        # Compare versions
-        return latest > current
-    except version.InvalidVersion:
-        logging.warning(f"Invalid version format: latest={latest_version}, current={current_version}")
+        # Validate version format
+        if len(version_parts) != 3:
+            raise ValueError(f"Invalid version format: {version}")
+
+        try:
+            # Convert version parts to integers
+            major, minor, patch = map(int, version_parts)
+            return (major, minor, patch, pre_release)
+        except ValueError:
+            raise ValueError(f"Invalid version format: {version}")
+
+    latest = parse_version(latest_version)
+    current = parse_version(current_version)
+
+    # Compare major, minor, and patch versions
+    for latest_part, current_part in zip(latest[:3], current[:3]):
+        if latest_part > current_part:
+            return True
+        elif latest_part < current_part:
+            return False
+
+    # If versions are equal, check pre-release suffixes
+    if latest[3] is None and current[3] is not None:
+        return True
+    elif latest[3] is not None and current[3] is None:
         return False
+    elif latest[3] is not None and current[3] is not None:
+        # Simple string comparison for pre-release versions
+        return latest[3] > current[3]
+
+    return False
 
 
 api.add_resource(VersionApi, "/version")

api/controllers/console/workspace/account.py

@@ -8,13 +8,14 @@ from flask_restful import Resource, fields, marshal_with, reqparse
 from configs import dify_config
 from constants.languages import supported_language
 from controllers.console import api
+from controllers.console.setup import setup_required
 from controllers.console.workspace.error import (
     AccountAlreadyInitedError,
     CurrentPasswordIncorrectError,
     InvalidInvitationCodeError,
     RepeatPasswordNotMatchError,
 )
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
 from fields.member_fields import account_fields
 from libs.helper import TimestampField, timezone

api/controllers/console/workspace/load_balancing_config.py

@@ -2,7 +2,8 @@ from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from libs.login import current_user, login_required

api/controllers/console/workspace/members.py

@@ -4,11 +4,8 @@ from flask_restful import Resource, abort, marshal_with, reqparse
 import services
 from configs import dify_config
 from controllers.console import api
-from controllers.console.wraps import (
-    account_initialization_required,
-    cloud_edition_billing_resource_check,
-    setup_required,
-)
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_database import db
 from fields.member_fields import account_with_role_list_fields
 from libs.login import login_required

api/controllers/console/workspace/model_providers.py

@@ -6,7 +6,8 @@ from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.utils.encoders import jsonable_encoder

api/controllers/console/workspace/models.py

@@ -5,7 +5,8 @@ from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.utils.encoders import jsonable_encoder

api/controllers/console/workspace/tool_providers.py

@@ -7,7 +7,8 @@ from werkzeug.exceptions import Forbidden
 
 from configs import dify_config
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.model_runtime.utils.encoders import jsonable_encoder
 from libs.helper import alphanumeric, uuid_value
 from libs.login import login_required

api/controllers/console/workspace/workspace.py

@@ -6,7 +6,6 @@ from flask_restful import Resource, fields, inputs, marshal, marshal_with, reqpa
 from werkzeug.exceptions import Unauthorized
 
 import services
-from controllers.common.errors import FilenameNotExistsError
 from controllers.console import api
 from controllers.console.admin import admin_required
 from controllers.console.datasets.error import (
@@ -16,11 +15,8 @@ from controllers.console.datasets.error import (
     UnsupportedFileTypeError,
 )
 from controllers.console.error import AccountNotLinkTenantError
-from controllers.console.wraps import (
-    account_initialization_required,
-    cloud_edition_billing_resource_check,
-    setup_required,
-)
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_database import db
 from libs.helper import TimestampField
 from libs.login import login_required
@@ -197,20 +193,12 @@ class WebappLogoWorkspaceApi(Resource):
         if len(request.files) > 1:
             raise TooManyFilesError()
 
-        if not file.filename:
-            raise FilenameNotExistsError
-
         extension = file.filename.split(".")[-1]
         if extension.lower() not in {"svg", "png"}:
             raise UnsupportedFileTypeError()
 
         try:
-            upload_file = FileService.upload_file(
-                filename=file.filename,
-                content=file.read(),
-                mimetype=file.mimetype,
-                user=current_user,
-            )
+            upload_file = FileService.upload_file(file=file, user=current_user)
 
         except services.errors.file.FileTooLargeError as file_too_large_error:
             raise FileTooLargeError(file_too_large_error.description)

api/controllers/console/wraps.py

@@ -1,5 +1,4 @@
 import json
-import os
 from functools import wraps
 
 from flask import abort, request
@@ -7,12 +6,9 @@ from flask_login import current_user
 
 from configs import dify_config
 from controllers.console.workspace.error import AccountNotInitializedError
-from models.model import DifySetup
 from services.feature_service import FeatureService
 from services.operation_service import OperationService
 
-from .error import NotInitValidateError, NotSetupError
-
 
 def account_initialization_required(view):
     @wraps(view)
@@ -128,17 +124,3 @@ def cloud_utm_record(view):
         return view(*args, **kwargs)
 
     return decorated
-
-
-def setup_required(view):
-    @wraps(view)
-    def decorated(*args, **kwargs):
-        # check setup
-        if dify_config.EDITION == "SELF_HOSTED" and os.environ.get("INIT_PASSWORD") and not DifySetup.query.first():
-            raise NotInitValidateError()
-        elif dify_config.EDITION == "SELF_HOSTED" and not DifySetup.query.first():
-            raise NotSetupError()
-
-        return view(*args, **kwargs)
-
-    return decorated

api/controllers/inner_api/workspace/workspace.py

@@ -1,6 +1,6 @@
 from flask_restful import Resource, reqparse
 
-from controllers.console.wraps import setup_required
+from controllers.console.setup import setup_required
 from controllers.inner_api import api
 from controllers.inner_api.wraps import inner_api_only
 from events.tenant_event import tenant_was_created

api/controllers/service_api/app/app.py

@@ -1,7 +1,6 @@
-from flask_restful import Resource, marshal_with
+from flask_restful import Resource, fields, marshal_with
 
-from controllers.common import fields
-from controllers.common import helpers as controller_helpers
+from configs import dify_config
 from controllers.service_api import api
 from controllers.service_api.app.error import AppUnavailableError
 from controllers.service_api.wraps import validate_app_token
@@ -12,8 +11,40 @@ from services.app_service import AppService
 class AppParameterApi(Resource):
     """Resource for app variables."""
 
+    variable_fields = {
+        "key": fields.String,
+        "name": fields.String,
+        "description": fields.String,
+        "type": fields.String,
+        "default": fields.String,
+        "max_length": fields.Integer,
+        "options": fields.List(fields.String),
+    }
+
+    system_parameters_fields = {
+        "image_file_size_limit": fields.Integer,
+        "video_file_size_limit": fields.Integer,
+        "audio_file_size_limit": fields.Integer,
+        "file_size_limit": fields.Integer,
+    }
+
+    parameters_fields = {
+        "opening_statement": fields.String,
+        "suggested_questions": fields.Raw,
+        "suggested_questions_after_answer": fields.Raw,
+        "speech_to_text": fields.Raw,
+        "text_to_speech": fields.Raw,
+        "retriever_resource": fields.Raw,
+        "annotation_reply": fields.Raw,
+        "more_like_this": fields.Raw,
+        "user_input_form": fields.Raw,
+        "sensitive_word_avoidance": fields.Raw,
+        "file_upload": fields.Raw,
+        "system_parameters": fields.Nested(system_parameters_fields),
+    }
+
     @validate_app_token
-    @marshal_with(fields.parameters_fields)
+    @marshal_with(parameters_fields)
     def get(self, app_model: App):
         """Retrieve app parameters."""
         if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}:
@@ -25,16 +56,43 @@ class AppParameterApi(Resource):
             user_input_form = workflow.user_input_form(to_old_structure=True)
         else:
             app_model_config = app_model.app_model_config
-            if app_model_config is None:
-                raise AppUnavailableError()
-
             features_dict = app_model_config.to_dict()
 
             user_input_form = features_dict.get("user_input_form", [])
 
-        return controller_helpers.get_parameters_from_feature_dict(
-            features_dict=features_dict, user_input_form=user_input_form
-        )
+        return {
+            "opening_statement": features_dict.get("opening_statement"),
+            "suggested_questions": features_dict.get("suggested_questions", []),
+            "suggested_questions_after_answer": features_dict.get(
+                "suggested_questions_after_answer", {"enabled": False}
+            ),
+            "speech_to_text": features_dict.get("speech_to_text", {"enabled": False}),
+            "text_to_speech": features_dict.get("text_to_speech", {"enabled": False}),
+            "retriever_resource": features_dict.get("retriever_resource", {"enabled": False}),
+            "annotation_reply": features_dict.get("annotation_reply", {"enabled": False}),
+            "more_like_this": features_dict.get("more_like_this", {"enabled": False}),
+            "user_input_form": user_input_form,
+            "sensitive_word_avoidance": features_dict.get(
+                "sensitive_word_avoidance", {"enabled": False, "type": "", "configs": []}
+            ),
+            "file_upload": features_dict.get(
+                "file_upload",
+                {
+                    "image": {
+                        "enabled": False,
+                        "number_limits": 3,
+                        "detail": "high",
+                        "transfer_methods": ["remote_url", "local_file"],
+                    }
+                },
+            ),
+            "system_parameters": {
+                "image_file_size_limit": dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT,
+                "video_file_size_limit": dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT,
+                "audio_file_size_limit": dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT,
+                "file_size_limit": dify_config.UPLOAD_FILE_SIZE_LIMIT,
+            },
+        }
 
 
 class AppMetaApi(Resource):

api/controllers/service_api/app/conversation.py

@@ -7,11 +7,7 @@ from controllers.service_api import api
 from controllers.service_api.app.error import NotChatAppError
 from controllers.service_api.wraps import FetchUserArg, WhereisUserArg, validate_app_token
 from core.app.entities.app_invoke_entities import InvokeFrom
-from fields.conversation_fields import (
-    conversation_delete_fields,
-    conversation_infinite_scroll_pagination_fields,
-    simple_conversation_fields,
-)
+from fields.conversation_fields import conversation_infinite_scroll_pagination_fields, simple_conversation_fields
 from libs.helper import uuid_value
 from models.model import App, AppMode, EndUser
 from services.conversation_service import ConversationService
@@ -53,7 +49,7 @@ class ConversationApi(Resource):
 
 class ConversationDetailApi(Resource):
     @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON))
-    @marshal_with(conversation_delete_fields)
+    @marshal_with(simple_conversation_fields)
     def delete(self, app_model: App, end_user: EndUser, c_id):
         app_mode = AppMode.value_of(app_model.mode)
         if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:

api/controllers/service_api/app/file.py

@@ -2,7 +2,6 @@ from flask import request
 from flask_restful import Resource, marshal_with
 
 import services
-from controllers.common.errors import FilenameNotExistsError
 from controllers.service_api import api
 from controllers.service_api.app.error import (
     FileTooLargeError,
@@ -32,16 +31,8 @@ class FileApi(Resource):
         if len(request.files) > 1:
             raise TooManyFilesError()
 
-        if not file.filename:
-            raise FilenameNotExistsError
-
         try:
-            upload_file = FileService.upload_file(
-                filename=file.filename,
-                content=file.read(),
-                mimetype=file.mimetype,
-                user=end_user,
-            )
+            upload_file = FileService.upload_file(file, end_user)
         except services.errors.file.FileTooLargeError as file_too_large_error:
             raise FileTooLargeError(file_too_large_error.description)
         except services.errors.file.UnsupportedFileTypeError:

api/controllers/service_api/app/message.py

@@ -10,7 +10,6 @@ from controllers.service_api.app.error import NotChatAppError
 from controllers.service_api.wraps import FetchUserArg, WhereisUserArg, validate_app_token
 from core.app.entities.app_invoke_entities import InvokeFrom
 from fields.conversation_fields import message_file_fields
-from fields.raws import FilesContainedField
 from libs.helper import TimestampField, uuid_value
 from models.model import App, AppMode, EndUser
 from services.errors.message import SuggestedQuestionsAfterAnswerDisabledError
@@ -56,7 +55,7 @@ class MessageListApi(Resource):
         "id": fields.String,
         "conversation_id": fields.String,
         "parent_message_id": fields.String,
-        "inputs": FilesContainedField,
+        "inputs": fields.Raw,
         "query": fields.String,
         "answer": fields.String(attribute="re_sign_file_url_answer"),
         "message_files": fields.List(fields.Nested(message_file_fields)),

api/controllers/service_api/dataset/document.py

@@ -6,7 +6,6 @@ from sqlalchemy import desc
 from werkzeug.exceptions import NotFound
 
 import services.dataset_service
-from controllers.common.errors import FilenameNotExistsError
 from controllers.service_api import api
 from controllers.service_api.app.error import ProviderNotInitializeError
 from controllers.service_api.dataset.error import (
@@ -56,12 +55,7 @@ class DocumentAddByTextApi(DatasetApiResource):
         if not dataset.indexing_technique and not args["indexing_technique"]:
             raise ValueError("indexing_technique is required.")
 
-        text = args.get("text")
-        name = args.get("name")
-        if text is None or name is None:
-            raise ValueError("Both 'text' and 'name' must be non-null values.")
-
-        upload_file = FileService.upload_text(text=str(text), text_name=str(name))
+        upload_file = FileService.upload_text(args.get("text"), args.get("name"))
         data_source = {
             "type": "upload_file",
             "info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": [upload_file.id]}},
@@ -110,11 +104,7 @@ class DocumentUpdateByTextApi(DatasetApiResource):
             raise ValueError("Dataset is not exist.")
 
         if args["text"]:
-            text = args.get("text")
-            name = args.get("name")
-            if text is None or name is None:
-                raise ValueError("Both text and name must be strings.")
-            upload_file = FileService.upload_text(text=str(text), text_name=str(name))
+            upload_file = FileService.upload_text(args.get("text"), args.get("name"))
             data_source = {
                 "type": "upload_file",
                 "info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": [upload_file.id]}},
@@ -173,16 +163,7 @@ class DocumentAddByFileApi(DatasetApiResource):
         if len(request.files) > 1:
             raise TooManyFilesError()
 
-        if not file.filename:
-            raise FilenameNotExistsError
-
-        upload_file = FileService.upload_file(
-            filename=file.filename,
-            content=file.read(),
-            mimetype=file.mimetype,
-            user=current_user,
-            source="datasets",
-        )
+        upload_file = FileService.upload_file(file, current_user)
         data_source = {"type": "upload_file", "info_list": {"file_info_list": {"file_ids": [upload_file.id]}}}
         args["data_source"] = data_source
         # validate args
@@ -231,16 +212,7 @@ class DocumentUpdateByFileApi(DatasetApiResource):
             if len(request.files) > 1:
                 raise TooManyFilesError()
 
-            if not file.filename:
-                raise FilenameNotExistsError
-
-            upload_file = FileService.upload_file(
-                filename=file.filename,
-                content=file.read(),
-                mimetype=file.mimetype,
-                user=current_user,
-                source="datasets",
-            )
+            upload_file = FileService.upload_file(file, current_user)
             data_source = {"type": "upload_file", "info_list": {"file_info_list": {"file_ids": [upload_file.id]}}}
             args["data_source"] = data_source
         # validate args
@@ -258,7 +230,7 @@ class DocumentUpdateByFileApi(DatasetApiResource):
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
         document = documents[0]
-        documents_and_batch_fields = {"document": marshal(document, document_fields), "batch": document.batch}
+        documents_and_batch_fields = {"document": marshal(document, document_fields), "batch": batch}
         return documents_and_batch_fields, 200
 
 
@@ -359,26 +331,10 @@ class DocumentIndexingStatusApi(DatasetApiResource):
         return data
 
 
-api.add_resource(
-    DocumentAddByTextApi,
-    "/datasets/<uuid:dataset_id>/document/create_by_text",
-    "/datasets/<uuid:dataset_id>/document/create-by-text",
-)
-api.add_resource(
-    DocumentAddByFileApi,
-    "/datasets/<uuid:dataset_id>/document/create_by_file",
-    "/datasets/<uuid:dataset_id>/document/create-by-file",
-)
-api.add_resource(
-    DocumentUpdateByTextApi,
-    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_text",
-    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update-by-text",
-)
-api.add_resource(
-    DocumentUpdateByFileApi,
-    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_file",
-    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update-by-file",
-)
+api.add_resource(DocumentAddByTextApi, "/datasets/<uuid:dataset_id>/document/create_by_text")
+api.add_resource(DocumentAddByFileApi, "/datasets/<uuid:dataset_id>/document/create_by_file")
+api.add_resource(DocumentUpdateByTextApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_text")
+api.add_resource(DocumentUpdateByFileApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_file")
 api.add_resource(DocumentDeleteApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
 api.add_resource(DocumentListApi, "/datasets/<uuid:dataset_id>/documents")
 api.add_resource(DocumentIndexingStatusApi, "/datasets/<uuid:dataset_id>/documents/<string:batch>/indexing-status")

api/controllers/service_api/dataset/hit_testing.py

@@ -14,4 +14,4 @@ class HitTestingApi(DatasetApiResource, DatasetsHitTestingBase):
         return self.perform_hit_testing(dataset, args)
 
 
-api.add_resource(HitTestingApi, "/datasets/<uuid:dataset_id>/hit-testing", "/datasets/<uuid:dataset_id>/retrieve")
+api.add_resource(HitTestingApi, "/datasets/<uuid:dataset_id>/hit-testing")

api/controllers/web/__init__.py

@@ -2,17 +2,8 @@ from flask import Blueprint
 
 from libs.external_api import ExternalApi
 
-from .files import FileApi
-from .remote_files import RemoteFileInfoApi, RemoteFileUploadApi
-
 bp = Blueprint("web", __name__, url_prefix="/api")
 api = ExternalApi(bp)
 
-# Files
-api.add_resource(FileApi, "/files/upload")
 
-# Remote files
-api.add_resource(RemoteFileInfoApi, "/remote-files/<path:url>")
-api.add_resource(RemoteFileUploadApi, "/remote-files/upload")
-
-from . import app, audio, completion, conversation, feature, message, passport, saved_message, site, workflow
+from . import app, audio, completion, conversation, feature, file, message, passport, saved_message, site, workflow

api/controllers/web/app.py

@@ -1,7 +1,6 @@
-from flask_restful import marshal_with
+from flask_restful import fields, marshal_with
 
-from controllers.common import fields
-from controllers.common import helpers as controller_helpers
+from configs import dify_config
 from controllers.web import api
 from controllers.web.error import AppUnavailableError
 from controllers.web.wraps import WebApiResource
@@ -12,7 +11,39 @@ from services.app_service import AppService
 class AppParameterApi(WebApiResource):
     """Resource for app variables."""
 
-    @marshal_with(fields.parameters_fields)
+    variable_fields = {
+        "key": fields.String,
+        "name": fields.String,
+        "description": fields.String,
+        "type": fields.String,
+        "default": fields.String,
+        "max_length": fields.Integer,
+        "options": fields.List(fields.String),
+    }
+
+    system_parameters_fields = {
+        "image_file_size_limit": fields.Integer,
+        "video_file_size_limit": fields.Integer,
+        "audio_file_size_limit": fields.Integer,
+        "file_size_limit": fields.Integer,
+    }
+
+    parameters_fields = {
+        "opening_statement": fields.String,
+        "suggested_questions": fields.Raw,
+        "suggested_questions_after_answer": fields.Raw,
+        "speech_to_text": fields.Raw,
+        "text_to_speech": fields.Raw,
+        "retriever_resource": fields.Raw,
+        "annotation_reply": fields.Raw,
+        "more_like_this": fields.Raw,
+        "user_input_form": fields.Raw,
+        "sensitive_word_avoidance": fields.Raw,
+        "file_upload": fields.Raw,
+        "system_parameters": fields.Nested(system_parameters_fields),
+    }
+
+    @marshal_with(parameters_fields)
     def get(self, app_model: App, end_user):
         """Retrieve app parameters."""
         if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}:
@@ -24,16 +55,43 @@ class AppParameterApi(WebApiResource):
             user_input_form = workflow.user_input_form(to_old_structure=True)
         else:
             app_model_config = app_model.app_model_config
-            if app_model_config is None:
-                raise AppUnavailableError()
-
             features_dict = app_model_config.to_dict()
 
             user_input_form = features_dict.get("user_input_form", [])
 
-        return controller_helpers.get_parameters_from_feature_dict(
-            features_dict=features_dict, user_input_form=user_input_form
-        )
+        return {
+            "opening_statement": features_dict.get("opening_statement"),
+            "suggested_questions": features_dict.get("suggested_questions", []),
+            "suggested_questions_after_answer": features_dict.get(
+                "suggested_questions_after_answer", {"enabled": False}
+            ),
+            "speech_to_text": features_dict.get("speech_to_text", {"enabled": False}),
+            "text_to_speech": features_dict.get("text_to_speech", {"enabled": False}),
+            "retriever_resource": features_dict.get("retriever_resource", {"enabled": False}),
+            "annotation_reply": features_dict.get("annotation_reply", {"enabled": False}),
+            "more_like_this": features_dict.get("more_like_this", {"enabled": False}),
+            "user_input_form": user_input_form,
+            "sensitive_word_avoidance": features_dict.get(
+                "sensitive_word_avoidance", {"enabled": False, "type": "", "configs": []}
+            ),
+            "file_upload": features_dict.get(
+                "file_upload",
+                {
+                    "image": {
+                        "enabled": False,
+                        "number_limits": 3,
+                        "detail": "high",
+                        "transfer_methods": ["remote_url", "local_file"],
+                    }
+                },
+            ),
+            "system_parameters": {
+                "image_file_size_limit": dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT,
+                "video_file_size_limit": dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT,
+                "audio_file_size_limit": dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT,
+                "file_size_limit": dify_config.UPLOAD_FILE_SIZE_LIMIT,
+            },
+        }
 
 
 class AppMeta(WebApiResource):

api/controllers/web/file.py

@@ -0,0 +1,56 @@
+import urllib.parse
+
+from flask import request
+from flask_restful import marshal_with, reqparse
+
+import services
+from controllers.web import api
+from controllers.web.error import FileTooLargeError, NoFileUploadedError, TooManyFilesError, UnsupportedFileTypeError
+from controllers.web.wraps import WebApiResource
+from core.helper import ssrf_proxy
+from fields.file_fields import file_fields, remote_file_info_fields
+from services.file_service import FileService
+
+
+class FileApi(WebApiResource):
+    @marshal_with(file_fields)
+    def post(self, app_model, end_user):
+        # get file from request
+        file = request.files["file"]
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("source", type=str, required=False, location="args")
+        source = parser.parse_args().get("source")
+
+        # check file
+        if "file" not in request.files:
+            raise NoFileUploadedError()
+
+        if len(request.files) > 1:
+            raise TooManyFilesError()
+        try:
+            upload_file = FileService.upload_file(file=file, user=end_user, source=source)
+        except services.errors.file.FileTooLargeError as file_too_large_error:
+            raise FileTooLargeError(file_too_large_error.description)
+        except services.errors.file.UnsupportedFileTypeError:
+            raise UnsupportedFileTypeError()
+
+        return upload_file, 201
+
+
+class RemoteFileInfoApi(WebApiResource):
+    @marshal_with(remote_file_info_fields)
+    def get(self, url):
+        decoded_url = urllib.parse.unquote(url)
+        try:
+            response = ssrf_proxy.head(decoded_url)
+            return {
+                "file_type": response.headers.get("Content-Type", "application/octet-stream"),
+                "file_length": int(response.headers.get("Content-Length", -1)),
+            }
+        except Exception as e:
+            return {"error": str(e)}, 400
+
+
+api.add_resource(FileApi, "/files/upload")
+api.add_resource(RemoteFileInfoApi, "/remote-files/<path:url>")

api/controllers/web/files.py

@@ -1,43 +0,0 @@
-from flask import request
-from flask_restful import marshal_with
-
-import services
-from controllers.common.errors import FilenameNotExistsError
-from controllers.web.error import FileTooLargeError, NoFileUploadedError, TooManyFilesError, UnsupportedFileTypeError
-from controllers.web.wraps import WebApiResource
-from fields.file_fields import file_fields
-from services.file_service import FileService
-
-
-class FileApi(WebApiResource):
-    @marshal_with(file_fields)
-    def post(self, app_model, end_user):
-        file = request.files["file"]
-        source = request.form.get("source")
-
-        if "file" not in request.files:
-            raise NoFileUploadedError()
-
-        if len(request.files) > 1:
-            raise TooManyFilesError()
-
-        if not file.filename:
-            raise FilenameNotExistsError
-
-        if source not in ("datasets", None):
-            source = None
-
-        try:
-            upload_file = FileService.upload_file(
-                filename=file.filename,
-                content=file.read(),
-                mimetype=file.mimetype,
-                user=end_user,
-                source=source,
-            )
-        except services.errors.file.FileTooLargeError as file_too_large_error:
-            raise FileTooLargeError(file_too_large_error.description)
-        except services.errors.file.UnsupportedFileTypeError:
-            raise UnsupportedFileTypeError()
-
-        return upload_file, 201

api/controllers/web/remote_files.py

@@ -1,75 +0,0 @@
-import urllib.parse
-
-import httpx
-from flask_restful import marshal_with, reqparse
-
-import services
-from controllers.common import helpers
-from controllers.web.wraps import WebApiResource
-from core.file import helpers as file_helpers
-from core.helper import ssrf_proxy
-from fields.file_fields import file_fields_with_signed_url, remote_file_info_fields
-from services.file_service import FileService
-
-from .error import FileTooLargeError, UnsupportedFileTypeError
-
-
-class RemoteFileInfoApi(WebApiResource):
-    @marshal_with(remote_file_info_fields)
-    def get(self, app_model, end_user, url):
-        decoded_url = urllib.parse.unquote(url)
-        resp = ssrf_proxy.head(decoded_url)
-        if resp.status_code != httpx.codes.OK:
-            # failed back to get method
-            resp = ssrf_proxy.get(decoded_url, timeout=3)
-        resp.raise_for_status()
-        return {
-            "file_type": resp.headers.get("Content-Type", "application/octet-stream"),
-            "file_length": int(resp.headers.get("Content-Length", -1)),
-        }
-
-
-class RemoteFileUploadApi(WebApiResource):
-    @marshal_with(file_fields_with_signed_url)
-    def post(self, app_model, end_user):  # Add app_model and end_user parameters
-        parser = reqparse.RequestParser()
-        parser.add_argument("url", type=str, required=True, help="URL is required")
-        args = parser.parse_args()
-
-        url = args["url"]
-
-        resp = ssrf_proxy.head(url=url)
-        if resp.status_code != httpx.codes.OK:
-            resp = ssrf_proxy.get(url=url, timeout=3)
-        resp.raise_for_status()
-
-        file_info = helpers.guess_file_info_from_response(resp)
-
-        if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):
-            raise FileTooLargeError
-
-        content = resp.content if resp.request.method == "GET" else ssrf_proxy.get(url).content
-
-        try:
-            upload_file = FileService.upload_file(
-                filename=file_info.filename,
-                content=content,
-                mimetype=file_info.mimetype,
-                user=end_user,
-                source_url=url,
-            )
-        except services.errors.file.FileTooLargeError as file_too_large_error:
-            raise FileTooLargeError(file_too_large_error.description)
-        except services.errors.file.UnsupportedFileTypeError:
-            raise UnsupportedFileTypeError
-
-        return {
-            "id": upload_file.id,
-            "name": upload_file.name,
-            "size": upload_file.size,
-            "extension": upload_file.extension,
-            "url": file_helpers.get_signed_file_url(upload_file_id=upload_file.id),
-            "mime_type": upload_file.mime_type,
-            "created_by": upload_file.created_by,
-            "created_at": upload_file.created_at,
-        }, 201

api/core/agent/base_agent_runner.py

@@ -30,7 +30,6 @@ from core.model_runtime.entities import (
     ToolPromptMessage,
     UserPromptMessage,
 )
-from core.model_runtime.entities.message_entities import ImagePromptMessageContent
 from core.model_runtime.entities.model_entities import ModelFeature
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.model_runtime.utils.encoders import jsonable_encoder
@@ -66,7 +65,7 @@ class BaseAgentRunner(AppRunner):
         prompt_messages: Optional[list[PromptMessage]] = None,
         variables_pool: Optional[ToolRuntimeVariablePool] = None,
         db_variables: Optional[ToolConversationVariables] = None,
-        model_instance: ModelInstance | None = None,
+        model_instance: ModelInstance = None,
     ) -> None:
         self.tenant_id = tenant_id
         self.application_generate_entity = application_generate_entity
@@ -509,27 +508,24 @@ class BaseAgentRunner(AppRunner):
 
     def organize_agent_user_prompt(self, message: Message) -> UserPromptMessage:
         files = db.session.query(MessageFile).filter(MessageFile.message_id == message.id).all()
-        if not files:
-            return UserPromptMessage(content=message.query)
-        file_extra_config = FileUploadConfigManager.convert(message.app_model_config.to_dict())
-        if not file_extra_config:
-            return UserPromptMessage(content=message.query)
+        if files:
+            file_extra_config = FileUploadConfigManager.convert(message.app_model_config.to_dict())
 
-        image_detail_config = file_extra_config.image_config.detail if file_extra_config.image_config else None
-        image_detail_config = image_detail_config or ImagePromptMessageContent.DETAIL.LOW
-
-        file_objs = file_factory.build_from_message_files(
-            message_files=files, tenant_id=self.tenant_id, config=file_extra_config
-        )
-        if not file_objs:
-            return UserPromptMessage(content=message.query)
-        prompt_message_contents: list[PromptMessageContent] = []
-        prompt_message_contents.append(TextPromptMessageContent(data=message.query))
-        for file in file_objs:
-            prompt_message_contents.append(
-                file_manager.to_prompt_message_content(
-                    file,
-                    image_detail_config=image_detail_config,
+            if file_extra_config:
+                file_objs = file_factory.build_from_message_files(
+                    message_files=files, tenant_id=self.tenant_id, config=file_extra_config
                 )
-            )
-        return UserPromptMessage(content=prompt_message_contents)
+            else:
+                file_objs = []
+
+            if not file_objs:
+                return UserPromptMessage(content=message.query)
+            else:
+                prompt_message_contents: list[PromptMessageContent] = []
+                prompt_message_contents.append(TextPromptMessageContent(data=message.query))
+                for file_obj in file_objs:
+                    prompt_message_contents.append(file_manager.to_prompt_message_content(file_obj))
+
+                return UserPromptMessage(content=prompt_message_contents)
+        else:
+            return UserPromptMessage(content=message.query)

api/core/agent/cot_chat_agent_runner.py

@@ -10,7 +10,6 @@ from core.model_runtime.entities import (
     TextPromptMessageContent,
     UserPromptMessage,
 )
-from core.model_runtime.entities.message_entities import ImagePromptMessageContent
 from core.model_runtime.utils.encoders import jsonable_encoder
 
 
@@ -37,24 +36,8 @@ class CotChatAgentRunner(CotAgentRunner):
         if self.files:
             prompt_message_contents: list[PromptMessageContent] = []
             prompt_message_contents.append(TextPromptMessageContent(data=query))
-
-            # get image detail config
-            image_detail_config = (
-                self.application_generate_entity.file_upload_config.image_config.detail
-                if (
-                    self.application_generate_entity.file_upload_config
-                    and self.application_generate_entity.file_upload_config.image_config
-                )
-                else None
-            )
-            image_detail_config = image_detail_config or ImagePromptMessageContent.DETAIL.LOW
-            for file in self.files:
-                prompt_message_contents.append(
-                    file_manager.to_prompt_message_content(
-                        file,
-                        image_detail_config=image_detail_config,
-                    )
-                )
+            for file_obj in self.files:
+                prompt_message_contents.append(file_manager.to_prompt_message_content(file_obj))
 
             prompt_messages.append(UserPromptMessage(content=prompt_message_contents))
         else:

api/core/agent/fc_agent_runner.py

@@ -22,7 +22,6 @@ from core.model_runtime.entities import (
     ToolPromptMessage,
     UserPromptMessage,
 )
-from core.model_runtime.entities.message_entities import ImagePromptMessageContent
 from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransform
 from core.tools.entities.tool_entities import ToolInvokeMeta
 from core.tools.tool_engine import ToolEngine
@@ -398,24 +397,8 @@ class FunctionCallAgentRunner(BaseAgentRunner):
         if self.files:
             prompt_message_contents: list[PromptMessageContent] = []
             prompt_message_contents.append(TextPromptMessageContent(data=query))
-
-            # get image detail config
-            image_detail_config = (
-                self.application_generate_entity.file_upload_config.image_config.detail
-                if (
-                    self.application_generate_entity.file_upload_config
-                    and self.application_generate_entity.file_upload_config.image_config
-                )
-                else None
-            )
-            image_detail_config = image_detail_config or ImagePromptMessageContent.DETAIL.LOW
-            for file in self.files:
-                prompt_message_contents.append(
-                    file_manager.to_prompt_message_content(
-                        file,
-                        image_detail_config=image_detail_config,
-                    )
-                )
+            for file_obj in self.files:
+                prompt_message_contents.append(file_manager.to_prompt_message_content(file_obj))
 
             prompt_messages.append(UserPromptMessage(content=prompt_message_contents))
         else:

api/core/app/app_config/entities.py

@@ -4,7 +4,7 @@ from typing import Any, Optional
 
 from pydantic import BaseModel, Field, field_validator
 
-from core.file import FileTransferMethod, FileType, FileUploadConfig
+from core.file import FileExtraConfig, FileTransferMethod, FileType
 from core.model_runtime.entities.message_entities import PromptMessageRole
 from models.model import AppMode
 
@@ -211,7 +211,7 @@ class TracingConfigEntity(BaseModel):
 
 
 class AppAdditionalFeatures(BaseModel):
-    file_upload: Optional[FileUploadConfig] = None
+    file_upload: Optional[FileExtraConfig] = None
     opening_statement: Optional[str] = None
     suggested_questions: list[str] = []
     suggested_questions_after_answer: bool = False

api/core/app/app_config/features/file_upload/manager.py

@@ -1,7 +1,8 @@
 from collections.abc import Mapping
 from typing import Any
 
-from core.file import FileUploadConfig
+from core.file.models import FileExtraConfig
+from models import FileUploadConfig
 
 
 class FileUploadConfigManager:
@@ -29,14 +30,15 @@ class FileUploadConfigManager:
                 if is_vision:
                     data["image_config"]["detail"] = file_upload_dict.get("image", {}).get("detail", "low")
 
-                return FileUploadConfig.model_validate(data)
+                return FileExtraConfig.model_validate(data)
 
     @classmethod
-    def validate_and_set_defaults(cls, config: dict) -> tuple[dict, list[str]]:
+    def validate_and_set_defaults(cls, config: dict, is_vision: bool = True) -> tuple[dict, list[str]]:
         """
         Validate and set defaults for file upload feature
 
         :param config: app model config args
+        :param is_vision: if True, the feature is vision feature
         """
         if not config.get("file_upload"):
             config["file_upload"] = {}

api/core/app/apps/advanced_chat/app_config_manager.py

@@ -52,7 +52,9 @@ class AdvancedChatAppConfigManager(BaseAppConfigManager):
         related_config_keys = []
 
         # file upload validation
-        config, current_related_config_keys = FileUploadConfigManager.validate_and_set_defaults(config=config)
+        config, current_related_config_keys = FileUploadConfigManager.validate_and_set_defaults(
+            config=config, is_vision=False
+        )
         related_config_keys.extend(current_related_config_keys)
 
         # opening_statement

api/core/app/apps/advanced_chat/app_generator.py

@@ -1,5 +1,6 @@
 import contextvars
 import logging
+import os
 import threading
 import uuid
 from collections.abc import Generator
@@ -9,7 +10,6 @@ from flask import Flask, current_app
 from pydantic import ValidationError
 
 import contexts
-from configs import dify_config
 from constants import UUID_NIL
 from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
 from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager
@@ -26,6 +26,7 @@ from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
 from factories import file_factory
 from models.account import Account
+from models.enums import CreatedByRole
 from models.model import App, Conversation, EndUser, Message
 from models.workflow import Workflow
 
@@ -97,10 +98,13 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
         # parse files
         files = args["files"] if args.get("files") else []
         file_extra_config = FileUploadConfigManager.convert(workflow.features_dict, is_vision=False)
+        role = CreatedByRole.ACCOUNT if isinstance(user, Account) else CreatedByRole.END_USER
         if file_extra_config:
             file_objs = file_factory.build_from_mappings(
                 mappings=files,
                 tenant_id=app_model.tenant_id,
+                user_id=user.id,
+                role=role,
                 config=file_extra_config,
             )
         else:
@@ -123,11 +127,10 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
         application_generate_entity = AdvancedChatAppGenerateEntity(
             task_id=str(uuid.uuid4()),
             app_config=app_config,
-            file_upload_config=file_extra_config,
             conversation_id=conversation.id if conversation else None,
             inputs=conversation.inputs
             if conversation
-            else self._prepare_user_inputs(user_inputs=inputs, app_config=app_config),
+            else self._prepare_user_inputs(user_inputs=inputs, app_config=app_config, user_id=user.id, role=role),
             query=query,
             files=file_objs,
             parent_message_id=args.get("parent_message_id") if invoke_from != InvokeFrom.SERVICE_API else UUID_NIL,
@@ -314,7 +317,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
                 logger.exception("Validation Error when generating")
                 queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER)
             except (ValueError, InvokeError) as e:
-                if dify_config.DEBUG:
+                if os.environ.get("DEBUG", "false").lower() == "true":
                     logger.exception("Error when generating")
                 queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER)
             except Exception as e:

api/core/app/apps/advanced_chat/generate_task_pipeline.py

@@ -20,7 +20,6 @@ from core.app.entities.queue_entities import (
     QueueIterationStartEvent,
     QueueMessageReplaceEvent,
     QueueNodeFailedEvent,
-    QueueNodeInIterationFailedEvent,
     QueueNodeStartedEvent,
     QueueNodeSucceededEvent,
     QueueParallelBranchRunFailedEvent,
@@ -242,7 +241,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                     start_listener_time = time.time()
                     yield MessageAudioStreamResponse(audio=audio_trunk.audio, task_id=task_id)
             except Exception as e:
-                logger.exception(e)
+                logger.error(e)
                 break
         if tts_publisher:
             yield MessageAudioEndStreamResponse(audio="", task_id=task_id)
@@ -315,7 +314,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
 
                 if response:
                     yield response
-            elif isinstance(event, QueueNodeFailedEvent | QueueNodeInIterationFailedEvent):
+            elif isinstance(event, QueueNodeFailedEvent):
                 workflow_node_execution = self._handle_workflow_node_execution_failed(event)
 
                 response = self._workflow_node_finish_to_stream_response(

api/core/app/apps/agent_chat/app_generator.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import threading
 import uuid
 from collections.abc import Generator
@@ -7,7 +8,6 @@ from typing import Any, Literal, Union, overload
 from flask import Flask, current_app
 from pydantic import ValidationError
 
-from configs import dify_config
 from constants import UUID_NIL
 from core.app.app_config.easy_ui_based_app.model_config.converter import ModelConfigConverter
 from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
@@ -23,6 +23,7 @@ from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
 from factories import file_factory
 from models import Account, App, EndUser
+from models.enums import CreatedByRole
 
 logger = logging.getLogger(__name__)
 
@@ -102,6 +103,8 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
             # always enable retriever resource in debugger mode
             override_model_config_dict["retriever_resource"] = {"enabled": True}
 
+        role = CreatedByRole.ACCOUNT if isinstance(user, Account) else CreatedByRole.END_USER
+
         # parse files
         files = args.get("files") or []
         file_extra_config = FileUploadConfigManager.convert(override_model_config_dict or app_model_config.to_dict())
@@ -109,6 +112,8 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
             file_objs = file_factory.build_from_mappings(
                 mappings=files,
                 tenant_id=app_model.tenant_id,
+                user_id=user.id,
+                role=role,
                 config=file_extra_config,
             )
         else:
@@ -130,11 +135,10 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
             task_id=str(uuid.uuid4()),
             app_config=app_config,
             model_conf=ModelConfigConverter.convert(app_config),
-            file_upload_config=file_extra_config,
             conversation_id=conversation.id if conversation else None,
             inputs=conversation.inputs
             if conversation
-            else self._prepare_user_inputs(user_inputs=inputs, app_config=app_config),
+            else self._prepare_user_inputs(user_inputs=inputs, app_config=app_config, user_id=user.id, role=role),
             query=query,
             files=file_objs,
             parent_message_id=args.get("parent_message_id") if invoke_from != InvokeFrom.SERVICE_API else UUID_NIL,
@@ -226,7 +230,7 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
                 logger.exception("Validation Error when generating")
                 queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER)
             except (ValueError, InvokeError) as e:
-                if dify_config.DEBUG:
+                if os.environ.get("DEBUG") and os.environ.get("DEBUG").lower() == "true":
                     logger.exception("Error when generating")
                 queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER)
             except Exception as e:

api/core/app/apps/base_app_generator.py

@@ -2,11 +2,12 @@ from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Optional
 
 from core.app.app_config.entities import VariableEntityType
-from core.file import File, FileUploadConfig
+from core.file import File, FileExtraConfig
 from factories import file_factory
 
 if TYPE_CHECKING:
     from core.app.app_config.entities import AppConfig, VariableEntity
+    from models.enums import CreatedByRole
 
 
 class BaseAppGenerator:
@@ -15,14 +16,13 @@ class BaseAppGenerator:
         *,
         user_inputs: Optional[Mapping[str, Any]],
         app_config: "AppConfig",
+        user_id: str,
+        role: "CreatedByRole",
     ) -> Mapping[str, Any]:
         user_inputs = user_inputs or {}
         # Filter input variables from form configuration, handle required fields, default values, and option values
         variables = app_config.variables
-        user_inputs = {
-            var.variable: self._validate_inputs(value=user_inputs.get(var.variable), variable_entity=var)
-            for var in variables
-        }
+        user_inputs = {var.variable: self._validate_input(inputs=user_inputs, var=var) for var in variables}
         user_inputs = {k: self._sanitize_value(v) for k, v in user_inputs.items()}
         # Convert files in inputs to File
         entity_dictionary = {item.variable: item for item in app_config.variables}
@@ -31,7 +31,9 @@ class BaseAppGenerator:
             k: file_factory.build_from_mapping(
                 mapping=v,
                 tenant_id=app_config.tenant_id,
-                config=FileUploadConfig(
+                user_id=user_id,
+                role=role,
+                config=FileExtraConfig(
                     allowed_file_types=entity_dictionary[k].allowed_file_types,
                     allowed_extensions=entity_dictionary[k].allowed_file_extensions,
                     allowed_upload_methods=entity_dictionary[k].allowed_file_upload_methods,
@@ -45,7 +47,9 @@ class BaseAppGenerator:
             k: file_factory.build_from_mappings(
                 mappings=v,
                 tenant_id=app_config.tenant_id,
-                config=FileUploadConfig(
+                user_id=user_id,
+                role=role,
+                config=FileExtraConfig(
                     allowed_file_types=entity_dictionary[k].allowed_file_types,
                     allowed_extensions=entity_dictionary[k].allowed_file_extensions,
                     allowed_upload_methods=entity_dictionary[k].allowed_file_upload_methods,
@@ -70,66 +74,50 @@ class BaseAppGenerator:
 
         return user_inputs
 
-    def _validate_inputs(
-        self,
-        *,
-        variable_entity: "VariableEntity",
-        value: Any,
-    ):
-        if value is None:
-            if variable_entity.required:
-                raise ValueError(f"{variable_entity.variable} is required in input form")
-            return value
+    def _validate_input(self, *, inputs: Mapping[str, Any], var: "VariableEntity"):
+        user_input_value = inputs.get(var.variable)
+        if not user_input_value:
+            if var.required:
+                raise ValueError(f"{var.variable} is required in input form")
+            else:
+                return None
 
-        if variable_entity.type in {
+        if var.type in {
             VariableEntityType.TEXT_INPUT,
             VariableEntityType.SELECT,
             VariableEntityType.PARAGRAPH,
-        } and not isinstance(value, str):
-            raise ValueError(
-                f"(type '{variable_entity.type}') {variable_entity.variable} in input form must be a string"
-            )
-
-        if variable_entity.type == VariableEntityType.NUMBER and isinstance(value, str):
+        } and not isinstance(user_input_value, str):
+            raise ValueError(f"(type '{var.type}') {var.variable} in input form must be a string")
+        if var.type == VariableEntityType.NUMBER and isinstance(user_input_value, str):
             # may raise ValueError if user_input_value is not a valid number
             try:
-                if "." in value:
-                    return float(value)
+                if "." in user_input_value:
+                    return float(user_input_value)
                 else:
-                    return int(value)
+                    return int(user_input_value)
             except ValueError:
-                raise ValueError(f"{variable_entity.variable} in input form must be a valid number")
+                raise ValueError(f"{var.variable} in input form must be a valid number")
+        if var.type == VariableEntityType.SELECT:
+            options = var.options
+            if user_input_value not in options:
+                raise ValueError(f"{var.variable} in input form must be one of the following: {options}")
+        elif var.type in {VariableEntityType.TEXT_INPUT, VariableEntityType.PARAGRAPH}:
+            if var.max_length and len(user_input_value) > var.max_length:
+                raise ValueError(f"{var.variable} in input form must be less than {var.max_length} characters")
+        elif var.type == VariableEntityType.FILE:
+            if not isinstance(user_input_value, dict) and not isinstance(user_input_value, File):
+                raise ValueError(f"{var.variable} in input form must be a file")
+        elif var.type == VariableEntityType.FILE_LIST:
+            if not (
+                isinstance(user_input_value, list)
+                and (
+                    all(isinstance(item, dict) for item in user_input_value)
+                    or all(isinstance(item, File) for item in user_input_value)
+                )
+            ):
+                raise ValueError(f"{var.variable} in input form must be a list of files")
 
-        match variable_entity.type:
-            case VariableEntityType.SELECT:
-                if value not in variable_entity.options:
-                    raise ValueError(
-                        f"{variable_entity.variable} in input form must be one of the following: "
-                        f"{variable_entity.options}"
-                    )
-            case VariableEntityType.TEXT_INPUT | VariableEntityType.PARAGRAPH:
-                if variable_entity.max_length and len(value) > variable_entity.max_length:
-                    raise ValueError(
-                        f"{variable_entity.variable} in input form must be less than {variable_entity.max_length} "
-                        "characters"
-                    )
-            case VariableEntityType.FILE:
-                if not isinstance(value, dict) and not isinstance(value, File):
-                    raise ValueError(f"{variable_entity.variable} in input form must be a file")
-            case VariableEntityType.FILE_LIST:
-                # if number of files exceeds the limit, raise ValueError
-                if not (
-                    isinstance(value, list)
-                    and (all(isinstance(item, dict) for item in value) or all(isinstance(item, File) for item in value))
-                ):
-                    raise ValueError(f"{variable_entity.variable} in input form must be a list of files")
-
-                if variable_entity.max_length and len(value) > variable_entity.max_length:
-                    raise ValueError(
-                        f"{variable_entity.variable} in input form must be less than {variable_entity.max_length} files"
-                    )
-
-        return value
+        return user_input_value
 
     def _sanitize_value(self, value: Any) -> Any:
         if isinstance(value, str):