chore(versioning): bump version to 0.9.0 (#8911)

Co-authored-by: JohnJyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>

.github/workflows/build-push.yml

@@ -125,7 +125,7 @@ jobs:
         with:
           images: ${{ env[matrix.image_name_env] }}
           tags: |
-            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') }}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') }}
             type=ref,event=branch
             type=sha,enable=true,priority=100,prefix=,suffix=,format=long
             type=raw,value=${{ github.ref_name }},enable=${{ startsWith(github.ref, 'refs/tags/') }}

.github/workflows/style.yml

@@ -20,7 +20,7 @@ jobs:
 
       - name: Check changed files
         id: changed-files
-        uses: tj-actions/changed-files@v44
+        uses: tj-actions/changed-files@v45
         with:
           files: api/**
 
@@ -66,7 +66,7 @@ jobs:
 
       - name: Check changed files
         id: changed-files
-        uses: tj-actions/changed-files@v44
+        uses: tj-actions/changed-files@v45
         with:
           files: web/**
 
@@ -97,7 +97,7 @@ jobs:
 
       - name: Check changed files
         id: changed-files
-        uses: tj-actions/changed-files@v44
+        uses: tj-actions/changed-files@v45
         with:
           files: |
             **.sh
@@ -107,7 +107,7 @@ jobs:
             dev/**
 
       - name: Super-linter
-        uses: super-linter/super-linter/slim@v6
+        uses: super-linter/super-linter/slim@v7
         if: steps.changed-files.outputs.any_changed == 'true'
         env:
           BASH_SEVERITY: warning

.github/workflows/translate-i18n-base-on-english.yml

@@ -0,0 +1,54 @@
+name: Check i18n Files and Create PR
+
+on:
+  pull_request:
+    types: [closed]
+    branches: [main]
+
+jobs:
+  check-and-update:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: web
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2 # last 2 commits
+
+      - name: Check for file changes in i18n/en-US
+        id: check_files
+        run: |
+          recent_commit_sha=$(git rev-parse HEAD)
+          second_recent_commit_sha=$(git rev-parse HEAD~1)
+          changed_files=$(git diff --name-only $recent_commit_sha $second_recent_commit_sha -- 'i18n/en-US/*.ts')
+          echo "Changed files: $changed_files"
+          if [ -n "$changed_files" ]; then
+            echo "FILES_CHANGED=true" >> $GITHUB_ENV
+          else
+            echo "FILES_CHANGED=false" >> $GITHUB_ENV
+          fi
+
+      - name: Set up Node.js
+        if: env.FILES_CHANGED == 'true'
+        uses: actions/setup-node@v2
+        with:
+          node-version: 'lts/*'
+
+      - name: Install dependencies
+        if: env.FILES_CHANGED == 'true'
+        run: yarn install --frozen-lockfile
+
+      - name: Run npm script
+        if: env.FILES_CHANGED == 'true'
+        run: npm run auto-gen-i18n
+
+      - name: Create Pull Request
+        if: env.FILES_CHANGED == 'true'
+        uses: peter-evans/create-pull-request@v6
+        with:
+          commit-message: Update i18n files based on en-US changes
+          title: 'chore: translate i18n files'
+          body: This PR was automatically created to update i18n files based on changes in en-US locale.
+          branch: chore/automated-i18n-updates

.github/workflows/web-tests.yml

@@ -0,0 +1,46 @@
+name: Web Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - web/**
+
+concurrency:
+  group: web-tests-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Web Tests
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./web
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Check changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v45
+        with:
+          files: web/**
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        if: steps.changed-files.outputs.any_changed == 'true'
+        with:
+          node-version: 20
+          cache: yarn
+          cache-dependency-path: ./web/package.json
+
+      - name: Install dependencies
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: yarn install --frozen-lockfile
+
+      - name: Run tests
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: yarn test

.gitignore

@@ -153,6 +153,9 @@ docker-legacy/volumes/etcd/*
 docker-legacy/volumes/minio/*
 docker-legacy/volumes/milvus/*
 docker-legacy/volumes/chroma/*
+docker-legacy/volumes/opensearch/data/*
+docker-legacy/volumes/pgvectors/data/*
+docker-legacy/volumes/pgvector/data/*
 
 docker/volumes/app/storage/*
 docker/volumes/certbot/*
@@ -164,6 +167,12 @@ docker/volumes/etcd/*
 docker/volumes/minio/*
 docker/volumes/milvus/*
 docker/volumes/chroma/*
+docker/volumes/opensearch/data/*
+docker/volumes/myscale/data/*
+docker/volumes/myscale/log/*
+docker/volumes/unstructured/*
+docker/volumes/pgvector/data/*
+docker/volumes/pgvecto_rs/data/*
 
 docker/nginx/conf.d/default.conf
 docker/middleware.env

CONTRIBUTING_CN.md

@@ -36,7 +36,7 @@
   | 被团队成员标记为高优先级的功能    | 高优先级   |
   | 在 [community feedback board](https://github.com/langgenius/dify/discussions/categories/feedbacks) 内反馈的常见功能请求 | 中等优先级 |
   | 非核心功能和小幅改进                     | 低优先级    |
-  | 有价值当不紧急                                   | 未来功能  |
+  | 有价值但不紧急                                   | 未来功能  |
 
 ### 其他任何事情（例如 bug 报告、性能优化、拼写错误更正）：
 * 立即开始编码。
@@ -138,7 +138,7 @@ Dify 的后端使用 Python 编写，使用 [Flask](https://flask.palletsproject
 ├── models                // 描述数据模型和 API 响应的形状
 ├── public                // 如 favicon 等元资源
 ├── service               // 定义 API 操作的形状
-├── test                  
+├── test
 ├── types                 // 函数参数和返回值的描述
 └── utils                 // 共享的实用函数
 ```

LICENSE

@@ -4,7 +4,7 @@ Dify is licensed under the Apache License 2.0, with the following additional con
 
 1. Dify may be utilized commercially, including as a backend service for other applications or as an application development platform for enterprises. Should the conditions below be met, a commercial license must be obtained from the producer:
 
-a. Multi-tenant SaaS service: Unless explicitly authorized by Dify in writing, you may not use the Dify source code to operate a multi-tenant environment. 
+a. Multi-tenant service: Unless explicitly authorized by Dify in writing, you may not use the Dify source code to operate a multi-tenant environment. 
     - Tenant Definition: Within the context of Dify, one tenant corresponds to one workspace. The workspace provides a separated area for each tenant's data and configurations.
 
 b. LOGO and copyright information: In the process of using Dify's frontend components, you may not remove or modify the LOGO or copyright information in the Dify console or applications. This restriction is inapplicable to uses of Dify that do not involve its frontend components.

api/.env.example

@@ -39,7 +39,7 @@ DB_DATABASE=dify
 
 # Storage configuration
 # use for store upload files, private keys...
-# storage type: local, s3, azure-blob, google-storage
+# storage type: local, s3, azure-blob, google-storage, tencent-cos, huawei-obs, volcengine-tos
 STORAGE_TYPE=local
 STORAGE_LOCAL_PATH=storage
 S3_USE_AWS_MANAGED_IAM=false
@@ -73,6 +73,12 @@ TENCENT_COS_SECRET_ID=your-secret-id
 TENCENT_COS_REGION=your-region
 TENCENT_COS_SCHEME=your-scheme
 
+# Huawei OBS Storage Configuration
+HUAWEI_OBS_BUCKET_NAME=your-bucket-name
+HUAWEI_OBS_SECRET_KEY=your-secret-key
+HUAWEI_OBS_ACCESS_KEY=your-access-key
+HUAWEI_OBS_SERVER=your-server-url
+
 # OCI Storage configuration
 OCI_ENDPOINT=your-endpoint
 OCI_BUCKET_NAME=your-bucket-name
@@ -80,6 +86,13 @@ OCI_ACCESS_KEY=your-access-key
 OCI_SECRET_KEY=your-secret-key
 OCI_REGION=your-region
 
+# Volcengine tos Storage configuration
+VOLCENGINE_TOS_ENDPOINT=your-endpoint
+VOLCENGINE_TOS_BUCKET_NAME=your-bucket-name
+VOLCENGINE_TOS_ACCESS_KEY=your-access-key
+VOLCENGINE_TOS_SECRET_KEY=your-secret-key
+VOLCENGINE_TOS_REGION=your-region
+
 # CORS configuration
 WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
@@ -101,11 +114,10 @@ QDRANT_GRPC_ENABLED=false
 QDRANT_GRPC_PORT=6334
 
 # Milvus configuration
-MILVUS_HOST=127.0.0.1
-MILVUS_PORT=19530
+MILVUS_URI=http://127.0.0.1:19530
+MILVUS_TOKEN=
 MILVUS_USER=root
 MILVUS_PASSWORD=Milvus
-MILVUS_SECURE=false
 
 # MyScale configuration
 MYSCALE_HOST=127.0.0.1
@@ -150,6 +162,8 @@ PGVECTOR_PORT=5433
 PGVECTOR_USER=postgres
 PGVECTOR_PASSWORD=postgres
 PGVECTOR_DATABASE=postgres
+PGVECTOR_MIN_CONNECTION=1
+PGVECTOR_MAX_CONNECTION=5
 
 # Tidb Vector configuration
 TIDB_VECTOR_HOST=xxx.eu-central-1.xxx.aws.tidbcloud.com

api/Dockerfile

@@ -55,7 +55,7 @@ RUN apt-get update \
     && echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \
     && apt-get update \
     # For Security
-    && apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1 expat=2.6.2-2 libldap-2.5-0=2.5.18+dfsg-3 perl=5.38.2-5 libsqlite3-0=3.46.0-1 \
+    && apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1 expat=2.6.3-1 libldap-2.5-0=2.5.18+dfsg-3 perl=5.38.2-5 libsqlite3-0=3.46.0-1 \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 

api/README.md

@@ -65,14 +65,12 @@
 
 8. Start Dify [web](../web) service.
 9. Setup your application by visiting `http://localhost:3000`...
-10. If you need to debug local async processing, please start the worker service.
+10. If you need to handle and debug the async tasks (e.g. dataset importing and documents indexing), please start the worker service.
 
    ```bash
    poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion
    ```
 
-   The started celery app handles the async tasks, e.g. dataset importing and documents indexing.
-
 ## Testing
 
 1. Install dependencies for both the backend and the test environment

api/app.py

@@ -53,11 +53,9 @@ from services.account_service import AccountService
 
 warnings.simplefilter("ignore", ResourceWarning)
 
-# fix windows platform
-if os.name == "nt":
-    os.system('tzutil /s "UTC"')
-else:
-    os.environ["TZ"] = "UTC"
+os.environ["TZ"] = "UTC"
+# windows platform not support tzset
+if hasattr(time, "tzset"):
     time.tzset()
 
 
@@ -164,7 +162,7 @@ def initialize_extensions(app):
 @login_manager.request_loader
 def load_user_from_request(request_from_flask_login):
     """Load user based on the request."""
-    if request.blueprint not in ["console", "inner_api"]:
+    if request.blueprint not in {"console", "inner_api"}:
         return None
     # Check if the user_id contains a dot, indicating the old format
     auth_header = request.headers.get("Authorization", "")

api/commands.py

@@ -28,28 +28,28 @@ from services.account_service import RegisterService, TenantService
 
 
 @click.command("reset-password", help="Reset the account password.")
-@click.option("--email", prompt=True, help="The email address of the account whose password you need to reset")
-@click.option("--new-password", prompt=True, help="the new password.")
-@click.option("--password-confirm", prompt=True, help="the new password confirm.")
+@click.option("--email", prompt=True, help="Account email to reset password for")
+@click.option("--new-password", prompt=True, help="New password")
+@click.option("--password-confirm", prompt=True, help="Confirm new password")
 def reset_password(email, new_password, password_confirm):
     """
     Reset password of owner account
     Only available in SELF_HOSTED mode
     """
     if str(new_password).strip() != str(password_confirm).strip():
-        click.echo(click.style("sorry. The two passwords do not match.", fg="red"))
+        click.echo(click.style("Passwords do not match.", fg="red"))
         return
 
     account = db.session.query(Account).filter(Account.email == email).one_or_none()
 
     if not account:
-        click.echo(click.style("sorry. the account: [{}] not exist .".format(email), fg="red"))
+        click.echo(click.style("Account not found for email: {}".format(email), fg="red"))
         return
 
     try:
         valid_password(new_password)
     except:
-        click.echo(click.style("sorry. The passwords must match {} ".format(password_pattern), fg="red"))
+        click.echo(click.style("Invalid password. Must match {}".format(password_pattern), fg="red"))
         return
 
     # generate password salt
@@ -62,37 +62,37 @@ def reset_password(email, new_password, password_confirm):
     account.password = base64_password_hashed
     account.password_salt = base64_salt
     db.session.commit()
-    click.echo(click.style("Congratulations! Password has been reset.", fg="green"))
+    click.echo(click.style("Password reset successfully.", fg="green"))
 
 
 @click.command("reset-email", help="Reset the account email.")
-@click.option("--email", prompt=True, help="The old email address of the account whose email you need to reset")
-@click.option("--new-email", prompt=True, help="the new email.")
-@click.option("--email-confirm", prompt=True, help="the new email confirm.")
+@click.option("--email", prompt=True, help="Current account email")
+@click.option("--new-email", prompt=True, help="New email")
+@click.option("--email-confirm", prompt=True, help="Confirm new email")
 def reset_email(email, new_email, email_confirm):
     """
     Replace account email
     :return:
     """
     if str(new_email).strip() != str(email_confirm).strip():
-        click.echo(click.style("Sorry, new email and confirm email do not match.", fg="red"))
+        click.echo(click.style("New emails do not match.", fg="red"))
         return
 
     account = db.session.query(Account).filter(Account.email == email).one_or_none()
 
     if not account:
-        click.echo(click.style("sorry. the account: [{}] not exist .".format(email), fg="red"))
+        click.echo(click.style("Account not found for email: {}".format(email), fg="red"))
         return
 
     try:
         email_validate(new_email)
     except:
-        click.echo(click.style("sorry. {} is not a valid email. ".format(email), fg="red"))
+        click.echo(click.style("Invalid email: {}".format(new_email), fg="red"))
         return
 
     account.email = new_email
     db.session.commit()
-    click.echo(click.style("Congratulations!, email has been reset.", fg="green"))
+    click.echo(click.style("Email updated successfully.", fg="green"))
 
 
 @click.command(
@@ -104,7 +104,7 @@ def reset_email(email, new_email, email_confirm):
 )
 @click.confirmation_option(
     prompt=click.style(
-        "Are you sure you want to reset encrypt key pair?" " this operation cannot be rolled back!", fg="red"
+        "Are you sure you want to reset encrypt key pair? This operation cannot be rolled back!", fg="red"
     )
 )
 def reset_encrypt_key_pair():
@@ -114,13 +114,13 @@ def reset_encrypt_key_pair():
     Only support SELF_HOSTED mode.
     """
     if dify_config.EDITION != "SELF_HOSTED":
-        click.echo(click.style("Sorry, only support SELF_HOSTED mode.", fg="red"))
+        click.echo(click.style("This command is only for SELF_HOSTED installations.", fg="red"))
         return
 
     tenants = db.session.query(Tenant).all()
     for tenant in tenants:
         if not tenant:
-            click.echo(click.style("Sorry, no workspace found. Please enter /install to initialize.", fg="red"))
+            click.echo(click.style("No workspaces found. Run /install first.", fg="red"))
             return
 
         tenant.encrypt_public_key = generate_key_pair(tenant.id)
@@ -131,18 +131,18 @@ def reset_encrypt_key_pair():
 
         click.echo(
             click.style(
-                "Congratulations! " "the asymmetric key pair of workspace {} has been reset.".format(tenant.id),
+                "Congratulations! The asymmetric key pair of workspace {} has been reset.".format(tenant.id),
                 fg="green",
             )
         )
 
 
-@click.command("vdb-migrate", help="migrate vector db.")
+@click.command("vdb-migrate", help="Migrate vector db.")
 @click.option("--scope", default="all", prompt=False, help="The scope of vector database to migrate, Default is All.")
 def vdb_migrate(scope: str):
-    if scope in ["knowledge", "all"]:
+    if scope in {"knowledge", "all"}:
         migrate_knowledge_vector_database()
-    if scope in ["annotation", "all"]:
+    if scope in {"annotation", "all"}:
         migrate_annotation_vector_database()
 
 
@@ -150,7 +150,7 @@ def migrate_annotation_vector_database():
     """
     Migrate annotation datas to target vector database .
     """
-    click.echo(click.style("Start migrate annotation data.", fg="green"))
+    click.echo(click.style("Starting annotation data migration.", fg="green"))
     create_count = 0
     skipped_count = 0
     total_count = 0
@@ -174,14 +174,14 @@ def migrate_annotation_vector_database():
                 f"Processing the {total_count} app {app.id}. " + f"{create_count} created, {skipped_count} skipped."
             )
             try:
-                click.echo("Create app annotation index: {}".format(app.id))
+                click.echo("Creating app annotation index: {}".format(app.id))
                 app_annotation_setting = (
                     db.session.query(AppAnnotationSetting).filter(AppAnnotationSetting.app_id == app.id).first()
                 )
 
                 if not app_annotation_setting:
                     skipped_count = skipped_count + 1
-                    click.echo("App annotation setting is disabled: {}".format(app.id))
+                    click.echo("App annotation setting disabled: {}".format(app.id))
                     continue
                 # get dataset_collection_binding info
                 dataset_collection_binding = (
@@ -190,7 +190,7 @@ def migrate_annotation_vector_database():
                     .first()
                 )
                 if not dataset_collection_binding:
-                    click.echo("App annotation collection binding is not exist: {}".format(app.id))
+                    click.echo("App annotation collection binding not found: {}".format(app.id))
                     continue
                 annotations = db.session.query(MessageAnnotation).filter(MessageAnnotation.app_id == app.id).all()
                 dataset = Dataset(
@@ -211,11 +211,11 @@ def migrate_annotation_vector_database():
                         documents.append(document)
 
                 vector = Vector(dataset, attributes=["doc_id", "annotation_id", "app_id"])
-                click.echo(f"Start to migrate annotation, app_id: {app.id}.")
+                click.echo(f"Migrating annotations for app: {app.id}.")
 
                 try:
                     vector.delete()
-                    click.echo(click.style(f"Successfully delete vector index for app: {app.id}.", fg="green"))
+                    click.echo(click.style(f"Deleted vector index for app {app.id}.", fg="green"))
                 except Exception as e:
                     click.echo(click.style(f"Failed to delete vector index for app {app.id}.", fg="red"))
                     raise e
@@ -223,12 +223,12 @@ def migrate_annotation_vector_database():
                     try:
                         click.echo(
                             click.style(
-                                f"Start to created vector index with {len(documents)} annotations for app {app.id}.",
+                                f"Creating vector index with {len(documents)} annotations for app {app.id}.",
                                 fg="green",
                             )
                         )
                         vector.create(documents)
-                        click.echo(click.style(f"Successfully created vector index for app {app.id}.", fg="green"))
+                        click.echo(click.style(f"Created vector index for app {app.id}.", fg="green"))
                     except Exception as e:
                         click.echo(click.style(f"Failed to created vector index for app {app.id}.", fg="red"))
                         raise e
@@ -237,14 +237,14 @@ def migrate_annotation_vector_database():
             except Exception as e:
                 click.echo(
                     click.style(
-                        "Create app annotation index error: {} {}".format(e.__class__.__name__, str(e)), fg="red"
+                        "Error creating app annotation index: {} {}".format(e.__class__.__name__, str(e)), fg="red"
                     )
                 )
                 continue
 
     click.echo(
         click.style(
-            f"Congratulations! Create {create_count} app annotation indexes, and skipped {skipped_count} apps.",
+            f"Migration complete. Created {create_count} app annotation indexes. Skipped {skipped_count} apps.",
             fg="green",
         )
     )
@@ -254,7 +254,7 @@ def migrate_knowledge_vector_database():
     """
     Migrate vector database datas to target vector database .
     """
-    click.echo(click.style("Start migrate vector db.", fg="green"))
+    click.echo(click.style("Starting vector database migration.", fg="green"))
     create_count = 0
     skipped_count = 0
     total_count = 0
@@ -275,11 +275,10 @@ def migrate_knowledge_vector_database():
         for dataset in datasets:
             total_count = total_count + 1
             click.echo(
-                f"Processing the {total_count} dataset {dataset.id}. "
-                + f"{create_count} created, {skipped_count} skipped."
+                f"Processing the {total_count} dataset {dataset.id}. {create_count} created, {skipped_count} skipped."
             )
             try:
-                click.echo("Create dataset vdb index: {}".format(dataset.id))
+                click.echo("Creating dataset vector database index: {}".format(dataset.id))
                 if dataset.index_struct_dict:
                     if dataset.index_struct_dict["type"] == vector_type:
                         skipped_count = skipped_count + 1
@@ -300,7 +299,7 @@ def migrate_knowledge_vector_database():
                         if dataset_collection_binding:
                             collection_name = dataset_collection_binding.collection_name
                         else:
-                            raise ValueError("Dataset Collection Bindings is not exist!")
+                            raise ValueError("Dataset Collection Binding not found")
                     else:
                         dataset_id = dataset.id
                         collection_name = Dataset.gen_collection_name_by_id(dataset_id)
@@ -352,14 +351,12 @@ def migrate_knowledge_vector_database():
                     raise ValueError(f"Vector store {vector_type} is not supported.")
 
                 vector = Vector(dataset)
-                click.echo(f"Start to migrate dataset {dataset.id}.")
+                click.echo(f"Migrating dataset {dataset.id}.")
 
                 try:
                     vector.delete()
                     click.echo(
-                        click.style(
-                            f"Successfully delete vector index {collection_name} for dataset {dataset.id}.", fg="green"
-                        )
+                        click.style(f"Deleted vector index {collection_name} for dataset {dataset.id}.", fg="green")
                     )
                 except Exception as e:
                     click.echo(
@@ -411,14 +408,13 @@ def migrate_knowledge_vector_database():
                     try:
                         click.echo(
                             click.style(
-                                f"Start to created vector index with {len(documents)} documents of {segments_count} segments for dataset {dataset.id}.",
+                                f"Creating vector index with {len(documents)} documents of {segments_count}"
+                                f" segments for dataset {dataset.id}.",
                                 fg="green",
                             )
                         )
                         vector.create(documents)
-                        click.echo(
-                            click.style(f"Successfully created vector index for dataset {dataset.id}.", fg="green")
-                        )
+                        click.echo(click.style(f"Created vector index for dataset {dataset.id}.", fg="green"))
                     except Exception as e:
                         click.echo(click.style(f"Failed to created vector index for dataset {dataset.id}.", fg="red"))
                         raise e
@@ -429,13 +425,13 @@ def migrate_knowledge_vector_database():
             except Exception as e:
                 db.session.rollback()
                 click.echo(
-                    click.style("Create dataset index error: {} {}".format(e.__class__.__name__, str(e)), fg="red")
+                    click.style("Error creating dataset index: {} {}".format(e.__class__.__name__, str(e)), fg="red")
                 )
                 continue
 
     click.echo(
         click.style(
-            f"Congratulations! Create {create_count} dataset indexes, and skipped {skipped_count} datasets.", fg="green"
+            f"Migration complete. Created {create_count} dataset indexes. Skipped {skipped_count} datasets.", fg="green"
         )
     )
 
@@ -445,7 +441,7 @@ def convert_to_agent_apps():
     """
     Convert Agent Assistant to Agent App.
     """
-    click.echo(click.style("Start convert to agent apps.", fg="green"))
+    click.echo(click.style("Starting convert to agent apps.", fg="green"))
 
     proceeded_app_ids = []
 
@@ -496,23 +492,23 @@ def convert_to_agent_apps():
             except Exception as e:
                 click.echo(click.style("Convert app error: {} {}".format(e.__class__.__name__, str(e)), fg="red"))
 
-    click.echo(click.style("Congratulations! Converted {} agent apps.".format(len(proceeded_app_ids)), fg="green"))
+    click.echo(click.style("Conversion complete. Converted {} agent apps.".format(len(proceeded_app_ids)), fg="green"))
 
 
-@click.command("add-qdrant-doc-id-index", help="add qdrant doc_id index.")
-@click.option("--field", default="metadata.doc_id", prompt=False, help="index field , default is metadata.doc_id.")
+@click.command("add-qdrant-doc-id-index", help="Add Qdrant doc_id index.")
+@click.option("--field", default="metadata.doc_id", prompt=False, help="Index field , default is metadata.doc_id.")
 def add_qdrant_doc_id_index(field: str):
-    click.echo(click.style("Start add qdrant doc_id index.", fg="green"))
+    click.echo(click.style("Starting Qdrant doc_id index creation.", fg="green"))
     vector_type = dify_config.VECTOR_STORE
     if vector_type != "qdrant":
-        click.echo(click.style("Sorry, only support qdrant vector store.", fg="red"))
+        click.echo(click.style("This command only supports Qdrant vector store.", fg="red"))
         return
     create_count = 0
 
     try:
         bindings = db.session.query(DatasetCollectionBinding).all()
         if not bindings:
-            click.echo(click.style("Sorry, no dataset collection bindings found.", fg="red"))
+            click.echo(click.style("No dataset collection bindings found.", fg="red"))
             return
         import qdrant_client
         from qdrant_client.http.exceptions import UnexpectedResponse
@@ -522,7 +518,7 @@ def add_qdrant_doc_id_index(field: str):
 
         for binding in bindings:
             if dify_config.QDRANT_URL is None:
-                raise ValueError("Qdrant url is required.")
+                raise ValueError("Qdrant URL is required.")
             qdrant_config = QdrantConfig(
                 endpoint=dify_config.QDRANT_URL,
                 api_key=dify_config.QDRANT_API_KEY,
@@ -539,41 +535,39 @@ def add_qdrant_doc_id_index(field: str):
             except UnexpectedResponse as e:
                 # Collection does not exist, so return
                 if e.status_code == 404:
-                    click.echo(
-                        click.style(f"Collection not found, collection_name:{binding.collection_name}.", fg="red")
-                    )
+                    click.echo(click.style(f"Collection not found: {binding.collection_name}.", fg="red"))
                     continue
                 # Some other error occurred, so re-raise the exception
                 else:
                     click.echo(
                         click.style(
-                            f"Failed to create qdrant index, collection_name:{binding.collection_name}.", fg="red"
+                            f"Failed to create Qdrant index for collection: {binding.collection_name}.", fg="red"
                         )
                     )
 
     except Exception as e:
-        click.echo(click.style("Failed to create qdrant client.", fg="red"))
+        click.echo(click.style("Failed to create Qdrant client.", fg="red"))
 
-    click.echo(click.style(f"Congratulations! Create {create_count} collection indexes.", fg="green"))
+    click.echo(click.style(f"Index creation complete. Created {create_count} collection indexes.", fg="green"))
 
 
 @click.command("create-tenant", help="Create account and tenant.")
-@click.option("--email", prompt=True, help="The email address of the tenant account.")
-@click.option("--name", prompt=True, help="The workspace name of the tenant account.")
+@click.option("--email", prompt=True, help="Tenant account email.")
+@click.option("--name", prompt=True, help="Workspace name.")
 @click.option("--language", prompt=True, help="Account language, default: en-US.")
 def create_tenant(email: str, language: Optional[str] = None, name: Optional[str] = None):
     """
     Create tenant account
     """
     if not email:
-        click.echo(click.style("Sorry, email is required.", fg="red"))
+        click.echo(click.style("Email is required.", fg="red"))
         return
 
     # Create account
     email = email.strip()
 
     if "@" not in email:
-        click.echo(click.style("Sorry, invalid email address.", fg="red"))
+        click.echo(click.style("Invalid email address.", fg="red"))
         return
 
     account_name = email.split("@")[0]
@@ -593,19 +587,19 @@ def create_tenant(email: str, language: Optional[str] = None, name: Optional[str
 
     click.echo(
         click.style(
-            "Congratulations! Account and tenant created.\n" "Account: {}\nPassword: {}".format(email, new_password),
+            "Account and tenant created.\nAccount: {}\nPassword: {}".format(email, new_password),
             fg="green",
         )
     )
 
 
-@click.command("upgrade-db", help="upgrade the database")
+@click.command("upgrade-db", help="Upgrade the database")
 def upgrade_db():
     click.echo("Preparing database migration...")
     lock = redis_client.lock(name="db_upgrade_lock", timeout=60)
     if lock.acquire(blocking=False):
         try:
-            click.echo(click.style("Start database migration.", fg="green"))
+            click.echo(click.style("Starting database migration.", fg="green"))
 
             # run db migration
             import flask_migrate
@@ -615,7 +609,7 @@ def upgrade_db():
             click.echo(click.style("Database migration successful!", fg="green"))
 
         except Exception as e:
-            logging.exception(f"Database migration failed, error: {e}")
+            logging.exception(f"Database migration failed: {e}")
         finally:
             lock.release()
     else:
@@ -627,7 +621,7 @@ def fix_app_site_missing():
     """
     Fix app related site missing issue.
     """
-    click.echo(click.style("Start fix app related site missing issue.", fg="green"))
+    click.echo(click.style("Starting fix for missing app-related sites.", fg="green"))
 
     failed_app_ids = []
     while True:
@@ -650,22 +644,22 @@ where sites.id is null limit 1000"""
                     if tenant:
                         accounts = tenant.get_accounts()
                         if not accounts:
-                            print("Fix app {} failed.".format(app.id))
+                            print("Fix failed for app {}".format(app.id))
                             continue
 
                         account = accounts[0]
-                        print("Fix app {} related site missing issue.".format(app.id))
+                        print("Fixing missing site for app {}".format(app.id))
                         app_was_created.send(app, account=account)
                 except Exception as e:
                     failed_app_ids.append(app_id)
-                    click.echo(click.style("Fix app {} related site missing issue failed!".format(app_id), fg="red"))
+                    click.echo(click.style("Failed to fix missing site for app {}".format(app_id), fg="red"))
                     logging.exception(f"Fix app related site missing issue failed, error: {e}")
                     continue
 
             if not processed_count:
                 break
 
-    click.echo(click.style("Congratulations! Fix app related site missing issue successful!", fg="green"))
+    click.echo(click.style("Fix for missing app-related sites completed successfully!", fg="green"))
 
 
 def register_commands(app):

api/configs/deploy/__init__.py

@@ -4,30 +4,30 @@ from pydantic_settings import BaseSettings
 
 class DeploymentConfig(BaseSettings):
     """
-    Deployment configs
+    Configuration settings for application deployment
     """
 
     APPLICATION_NAME: str = Field(
-        description="application name",
+        description="Name of the application, used for identification and logging purposes",
         default="langgenius/dify",
     )
 
     DEBUG: bool = Field(
-        description="whether to enable debug mode.",
+        description="Enable debug mode for additional logging and development features",
         default=False,
     )
 
     TESTING: bool = Field(
-        description="",
+        description="Enable testing mode for running automated tests",
         default=False,
     )
 
     EDITION: str = Field(
-        description="deployment edition",
+        description="Deployment edition of the application (e.g., 'SELF_HOSTED', 'CLOUD')",
         default="SELF_HOSTED",
     )
 
     DEPLOY_ENV: str = Field(
-        description="deployment environment, default to PRODUCTION.",
+        description="Deployment environment (e.g., 'PRODUCTION', 'DEVELOPMENT'), default to PRODUCTION",
         default="PRODUCTION",
     )

api/configs/enterprise/__init__.py

@@ -4,17 +4,17 @@ from pydantic_settings import BaseSettings
 
 class EnterpriseFeatureConfig(BaseSettings):
     """
-    Enterprise feature configs.
+    Configuration for enterprise-level features.
     **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
     """
 
     ENTERPRISE_ENABLED: bool = Field(
-        description="whether to enable enterprise features."
+        description="Enable or disable enterprise-level features."
         "Before using, please contact business@dify.ai by email to inquire about licensing matters.",
         default=False,
     )
 
     CAN_REPLACE_LOGO: bool = Field(
-        description="whether to allow replacing enterprise logo.",
+        description="Allow customization of the enterprise logo.",
         default=False,
     )

api/configs/extra/notion_config.py

@@ -6,30 +6,31 @@ from pydantic_settings import BaseSettings
 
 class NotionConfig(BaseSettings):
     """
-    Notion integration configs
+    Configuration settings for Notion integration
     """
 
     NOTION_CLIENT_ID: Optional[str] = Field(
-        description="Notion client ID",
+        description="Client ID for Notion API authentication. Required for OAuth 2.0 flow.",
         default=None,
     )
 
     NOTION_CLIENT_SECRET: Optional[str] = Field(
-        description="Notion client secret key",
+        description="Client secret for Notion API authentication. Required for OAuth 2.0 flow.",
         default=None,
     )
 
     NOTION_INTEGRATION_TYPE: Optional[str] = Field(
-        description="Notion integration type, default to None, available values: internal.",
+        description="Type of Notion integration."
+        " Set to 'internal' for internal integrations, or None for public integrations.",
         default=None,
     )
 
     NOTION_INTERNAL_SECRET: Optional[str] = Field(
-        description="Notion internal secret key",
+        description="Secret key for internal Notion integrations. Required when NOTION_INTEGRATION_TYPE is 'internal'.",
         default=None,
     )
 
     NOTION_INTEGRATION_TOKEN: Optional[str] = Field(
-        description="Notion integration token",
+        description="Integration token for Notion API access. Used for direct API calls without OAuth flow.",
         default=None,
     )

api/configs/extra/sentry_config.py

@@ -6,20 +6,23 @@ from pydantic_settings import BaseSettings
 
 class SentryConfig(BaseSettings):
     """
-    Sentry configs
+    Configuration settings for Sentry error tracking and performance monitoring
     """
 
     SENTRY_DSN: Optional[str] = Field(
-        description="Sentry DSN",
+        description="Sentry Data Source Name (DSN)."
+        " This is the unique identifier of your Sentry project, used to send events to the correct project.",
         default=None,
     )
 
     SENTRY_TRACES_SAMPLE_RATE: NonNegativeFloat = Field(
-        description="Sentry trace sample rate",
+        description="Sample rate for Sentry performance monitoring traces."
+        " Value between 0.0 and 1.0, where 1.0 means 100% of traces are sent to Sentry.",
         default=1.0,
     )
 
     SENTRY_PROFILES_SAMPLE_RATE: NonNegativeFloat = Field(
-        description="Sentry profiles sample rate",
+        description="Sample rate for Sentry profiling."
+        " Value between 0.0 and 1.0, where 1.0 means 100% of profiles are sent to Sentry.",
         default=1.0,
     )

api/configs/feature/__init__.py

@@ -8,145 +8,143 @@ from configs.feature.hosted_service import HostedServiceConfig
 
 class SecurityConfig(BaseSettings):
     """
-    Secret Key configs
+    Security-related configurations for the application
     """
 
     SECRET_KEY: Optional[str] = Field(
-        description="Your App secret key will be used for securely signing the session cookie"
+        description="Secret key for secure session cookie signing."
         "Make sure you are changing this key for your deployment with a strong key."
-        "You can generate a strong key using `openssl rand -base64 42`."
-        "Alternatively you can set it with `SECRET_KEY` environment variable.",
+        "Generate a strong key using `openssl rand -base64 42` or set via the `SECRET_KEY` environment variable.",
         default=None,
     )
 
     RESET_PASSWORD_TOKEN_EXPIRY_HOURS: PositiveInt = Field(
-        description="Expiry time in hours for reset token",
+        description="Duration in hours for which a password reset token remains valid",
         default=24,
     )
 
 
 class AppExecutionConfig(BaseSettings):
     """
-    App Execution configs
+    Configuration parameters for application execution
     """
 
     APP_MAX_EXECUTION_TIME: PositiveInt = Field(
-        description="execution timeout in seconds for app execution",
+        description="Maximum allowed execution time for the application in seconds",
         default=1200,
     )
     APP_MAX_ACTIVE_REQUESTS: NonNegativeInt = Field(
-        description="max active request per app, 0 means unlimited",
+        description="Maximum number of concurrent active requests per app (0 for unlimited)",
         default=0,
     )
 
 
 class CodeExecutionSandboxConfig(BaseSettings):
     """
-    Code Execution Sandbox configs
+    Configuration for the code execution sandbox environment
     """
 
     CODE_EXECUTION_ENDPOINT: HttpUrl = Field(
-        description="endpoint URL of code execution servcie",
+        description="URL endpoint for the code execution service",
         default="http://sandbox:8194",
     )
 
     CODE_EXECUTION_API_KEY: str = Field(
-        description="API key for code execution service",
+        description="API key for accessing the code execution service",
         default="dify-sandbox",
     )
 
     CODE_EXECUTION_CONNECT_TIMEOUT: Optional[float] = Field(
-        description="connect timeout in seconds for code execution request",
+        description="Connection timeout in seconds for code execution requests",
         default=10.0,
     )
 
     CODE_EXECUTION_READ_TIMEOUT: Optional[float] = Field(
-        description="read timeout in seconds for code execution request",
+        description="Read timeout in seconds for code execution requests",
         default=60.0,
     )
 
     CODE_EXECUTION_WRITE_TIMEOUT: Optional[float] = Field(
-        description="write timeout in seconds for code execution request",
+        description="Write timeout in seconds for code execution request",
         default=10.0,
     )
 
     CODE_MAX_NUMBER: PositiveInt = Field(
-        description="max depth for code execution",
+        description="Maximum allowed numeric value in code execution",
         default=9223372036854775807,
     )
 
     CODE_MIN_NUMBER: NegativeInt = Field(
-        description="",
+        description="Minimum allowed numeric value in code execution",
         default=-9223372036854775807,
     )
 
     CODE_MAX_DEPTH: PositiveInt = Field(
-        description="max depth for code execution",
+        description="Maximum allowed depth for nested structures in code execution",
         default=5,
     )
 
     CODE_MAX_PRECISION: PositiveInt = Field(
-        description="max precision digits for float type in code execution",
+        description="mMaximum number of decimal places for floating-point numbers in code execution",
         default=20,
     )
 
     CODE_MAX_STRING_LENGTH: PositiveInt = Field(
-        description="max string length for code execution",
+        description="Maximum allowed length for strings in code execution",
         default=80000,
     )
 
     CODE_MAX_STRING_ARRAY_LENGTH: PositiveInt = Field(
-        description="",
+        description="Maximum allowed length for string arrays in code execution",
         default=30,
     )
 
     CODE_MAX_OBJECT_ARRAY_LENGTH: PositiveInt = Field(
-        description="",
+        description="Maximum allowed length for object arrays in code execution",
         default=30,
     )
 
     CODE_MAX_NUMBER_ARRAY_LENGTH: PositiveInt = Field(
-        description="",
+        description="Maximum allowed length for numeric arrays in code execution",
         default=1000,
     )
 
 
 class EndpointConfig(BaseSettings):
     """
-    Module URL configs
+    Configuration for various application endpoints and URLs
     """
 
     CONSOLE_API_URL: str = Field(
-        description="The backend URL prefix of the console API."
-        "used to concatenate the login authorization callback or notion integration callback.",
+        description="Base URL for the console API,"
+        "used for login authentication callback or notion integration callbacks",
         default="",
     )
 
     CONSOLE_WEB_URL: str = Field(
-        description="The front-end URL prefix of the console web."
-        "used to concatenate some front-end addresses and for CORS configuration use.",
+        description="Base URL for the console web interface," "used for frontend references and CORS configuration",
         default="",
     )
 
     SERVICE_API_URL: str = Field(
-        description="Service API Url prefix." "used to display Service API Base Url to the front-end.",
+        description="Base URL for the service API, displayed to users for API access",
         default="",
     )
 
     APP_WEB_URL: str = Field(
-        description="WebApp Url prefix." "used to display WebAPP API Base Url to the front-end.",
+        description="Base URL for the web application, used for frontend references",
         default="",
     )
 
 
 class FileAccessConfig(BaseSettings):
     """
-    File Access configs
+    Configuration for file access and handling
     """
 
     FILES_URL: str = Field(
-        description="File preview or download Url prefix."
-        " used to display File preview or download Url to the front-end or as Multi-model inputs;"
+        description="Base URL for file preview or download,"
+        " used for frontend display and multi-model inputs"
         "Url is signed and has expiration time.",
         validation_alias=AliasChoices("FILES_URL", "CONSOLE_API_URL"),
         alias_priority=1,
@@ -154,49 +152,49 @@ class FileAccessConfig(BaseSettings):
     )
 
     FILES_ACCESS_TIMEOUT: int = Field(
-        description="timeout in seconds for file accessing",
+        description="Expiration time in seconds for file access URLs",
         default=300,
     )
 
 
 class FileUploadConfig(BaseSettings):
     """
-    File Uploading configs
+    Configuration for file upload limitations
     """
 
     UPLOAD_FILE_SIZE_LIMIT: NonNegativeInt = Field(
-        description="size limit in Megabytes for uploading files",
+        description="Maximum allowed file size for uploads in megabytes",
         default=15,
     )
 
     UPLOAD_FILE_BATCH_LIMIT: NonNegativeInt = Field(
-        description="batch size limit for uploading files",
+        description="Maximum number of files allowed in a single upload batch",
         default=5,
     )
 
     UPLOAD_IMAGE_FILE_SIZE_LIMIT: NonNegativeInt = Field(
-        description="image file size limit in Megabytes for uploading files",
+        description="Maximum allowed image file size for uploads in megabytes",
         default=10,
     )
 
     BATCH_UPLOAD_LIMIT: NonNegativeInt = Field(
-        description="",  # todo: to be clarified
+        description="Maximum number of files allowed in a batch upload operation",
         default=20,
     )
 
 
 class HttpConfig(BaseSettings):
     """
-    HTTP configs
+    HTTP-related configurations for the application
     """
 
     API_COMPRESSION_ENABLED: bool = Field(
-        description="whether to enable HTTP response compression of gzip",
+        description="Enable or disable gzip compression for HTTP responses",
         default=False,
     )
 
     inner_CONSOLE_CORS_ALLOW_ORIGINS: str = Field(
-        description="",
+        description="Comma-separated list of allowed origins for CORS in the console",
         validation_alias=AliasChoices("CONSOLE_CORS_ALLOW_ORIGINS", "CONSOLE_WEB_URL"),
         default="",
     )
@@ -218,359 +216,360 @@ class HttpConfig(BaseSettings):
         return self.inner_WEB_API_CORS_ALLOW_ORIGINS.split(",")
 
     HTTP_REQUEST_MAX_CONNECT_TIMEOUT: Annotated[
-        PositiveInt, Field(ge=10, description="connect timeout in seconds for HTTP request")
+        PositiveInt, Field(ge=10, description="Maximum connection timeout in seconds for HTTP requests")
     ] = 10
 
     HTTP_REQUEST_MAX_READ_TIMEOUT: Annotated[
-        PositiveInt, Field(ge=60, description="read timeout in seconds for HTTP request")
+        PositiveInt, Field(ge=60, description="Maximum read timeout in seconds for HTTP requests")
     ] = 60
 
     HTTP_REQUEST_MAX_WRITE_TIMEOUT: Annotated[
-        PositiveInt, Field(ge=10, description="read timeout in seconds for HTTP request")
+        PositiveInt, Field(ge=10, description="Maximum write timeout in seconds for HTTP requests")
     ] = 20
 
     HTTP_REQUEST_NODE_MAX_BINARY_SIZE: PositiveInt = Field(
-        description="",
+        description="Maximum allowed size in bytes for binary data in HTTP requests",
         default=10 * 1024 * 1024,
     )
 
     HTTP_REQUEST_NODE_MAX_TEXT_SIZE: PositiveInt = Field(
-        description="",
+        description="Maximum allowed size in bytes for text data in HTTP requests",
         default=1 * 1024 * 1024,
     )
 
     SSRF_PROXY_HTTP_URL: Optional[str] = Field(
-        description="HTTP URL for SSRF proxy",
+        description="Proxy URL for HTTP requests to prevent Server-Side Request Forgery (SSRF)",
         default=None,
     )
 
     SSRF_PROXY_HTTPS_URL: Optional[str] = Field(
-        description="HTTPS URL for SSRF proxy",
+        description="Proxy URL for HTTPS requests to prevent Server-Side Request Forgery (SSRF)",
         default=None,
     )
 
 
 class InnerAPIConfig(BaseSettings):
     """
-    Inner API configs
+    Configuration for internal API functionality
     """
 
     INNER_API: bool = Field(
-        description="whether to enable the inner API",
+        description="Enable or disable the internal API",
         default=False,
     )
 
     INNER_API_KEY: Optional[str] = Field(
-        description="The inner API key is used to authenticate the inner API",
+        description="API key for accessing the internal API",
         default=None,
     )
 
 
 class LoggingConfig(BaseSettings):
     """
-    Logging configs
+    Configuration for application logging
     """
 
     LOG_LEVEL: str = Field(
-        description="Log output level, default to INFO." "It is recommended to set it to ERROR for production.",
+        description="Logging level, default to INFO. Set to ERROR for production environments.",
         default="INFO",
     )
 
     LOG_FILE: Optional[str] = Field(
-        description="logging output file path",
+        description="File path for log output.",
         default=None,
     )
 
     LOG_FORMAT: str = Field(
-        description="log format",
+        description="Format string for log messages",
         default="%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s",
     )
 
     LOG_DATEFORMAT: Optional[str] = Field(
-        description="log date format",
+        description="Date format string for log timestamps",
         default=None,
     )
 
     LOG_TZ: Optional[str] = Field(
-        description="specify log timezone, eg: America/New_York",
+        description="Timezone for log timestamps (e.g., 'America/New_York')",
         default=None,
     )
 
 
 class ModelLoadBalanceConfig(BaseSettings):
     """
-    Model load balance configs
+    Configuration for model load balancing
     """
 
     MODEL_LB_ENABLED: bool = Field(
-        description="whether to enable model load balancing",
+        description="Enable or disable load balancing for models",
         default=False,
     )
 
 
 class BillingConfig(BaseSettings):
     """
-    Platform Billing Configurations
+    Configuration for platform billing features
     """
 
     BILLING_ENABLED: bool = Field(
-        description="whether to enable billing",
+        description="Enable or disable billing functionality",
         default=False,
     )
 
 
 class UpdateConfig(BaseSettings):
     """
-    Update configs
+    Configuration for application update checks
     """
 
     CHECK_UPDATE_URL: str = Field(
-        description="url for checking updates",
+        description="URL to check for application updates",
         default="https://updates.dify.ai",
     )
 
 
 class WorkflowConfig(BaseSettings):
     """
-    Workflow feature configs
+    Configuration for workflow execution
     """
 
     WORKFLOW_MAX_EXECUTION_STEPS: PositiveInt = Field(
-        description="max execution steps in single workflow execution",
+        description="Maximum number of steps allowed in a single workflow execution",
         default=500,
     )
 
     WORKFLOW_MAX_EXECUTION_TIME: PositiveInt = Field(
-        description="max execution time in seconds in single workflow execution",
+        description="Maximum execution time in seconds for a single workflow",
         default=1200,
     )
 
     WORKFLOW_CALL_MAX_DEPTH: PositiveInt = Field(
-        description="max depth of calling in single workflow execution",
+        description="Maximum allowed depth for nested workflow calls",
         default=5,
     )
 
     MAX_VARIABLE_SIZE: PositiveInt = Field(
-        description="The maximum size in bytes of a variable. default to 5KB.",
+        description="Maximum size in bytes for a single variable in workflows. Default to 5KB.",
         default=5 * 1024,
     )
 
 
 class OAuthConfig(BaseSettings):
     """
-    oauth configs
+    Configuration for OAuth authentication
     """
 
     OAUTH_REDIRECT_PATH: str = Field(
-        description="redirect path for OAuth",
+        description="Redirect path for OAuth authentication callbacks",
         default="/console/api/oauth/authorize",
     )
 
     GITHUB_CLIENT_ID: Optional[str] = Field(
-        description="GitHub client id for OAuth",
+        description="GitHub OAuth client secret",
         default=None,
     )
 
     GITHUB_CLIENT_SECRET: Optional[str] = Field(
-        description="GitHub client secret key for OAuth",
+        description="GitHub OAuth client secret",
         default=None,
     )
 
     GOOGLE_CLIENT_ID: Optional[str] = Field(
-        description="Google client id for OAuth",
+        description="Google OAuth client ID",
         default=None,
     )
 
     GOOGLE_CLIENT_SECRET: Optional[str] = Field(
-        description="Google client secret key for OAuth",
+        description="Google OAuth client secret",
         default=None,
     )
 
 
 class ModerationConfig(BaseSettings):
     """
-    Moderation in app configs.
+    Configuration for content moderation
     """
 
     MODERATION_BUFFER_SIZE: PositiveInt = Field(
-        description="buffer size for moderation",
+        description="Size of the buffer for content moderation processing",
         default=300,
     )
 
 
 class ToolConfig(BaseSettings):
     """
-    Tool configs
+    Configuration for tool management
     """
 
     TOOL_ICON_CACHE_MAX_AGE: PositiveInt = Field(
-        description="max age in seconds for tool icon caching",
+        description="Maximum age in seconds for caching tool icons",
         default=3600,
     )
 
 
 class MailConfig(BaseSettings):
     """
-    Mail Configurations
+    Configuration for email services
     """
 
     MAIL_TYPE: Optional[str] = Field(
-        description="Mail provider type name, default to None, availabile values are `smtp` and `resend`.",
+        description="Email service provider type ('smtp' or 'resend'), default to None.",
         default=None,
     )
 
     MAIL_DEFAULT_SEND_FROM: Optional[str] = Field(
-        description="default email address for sending from ",
+        description="Default email address to use as the sender",
         default=None,
     )
 
     RESEND_API_KEY: Optional[str] = Field(
-        description="API key for Resend",
+        description="API key for Resend email service",
         default=None,
     )
 
     RESEND_API_URL: Optional[str] = Field(
-        description="API URL for Resend",
+        description="API URL for Resend email service",
         default=None,
     )
 
     SMTP_SERVER: Optional[str] = Field(
-        description="smtp server host",
+        description="SMTP server hostname",
         default=None,
     )
 
     SMTP_PORT: Optional[int] = Field(
-        description="smtp server port",
+        description="SMTP server port number",
         default=465,
     )
 
     SMTP_USERNAME: Optional[str] = Field(
-        description="smtp server username",
+        description="Username for SMTP authentication",
         default=None,
     )
 
     SMTP_PASSWORD: Optional[str] = Field(
-        description="smtp server password",
+        description="Password for SMTP authentication",
         default=None,
     )
 
     SMTP_USE_TLS: bool = Field(
-        description="whether to use TLS connection to smtp server",
+        description="Enable TLS encryption for SMTP connections",
         default=False,
     )
 
     SMTP_OPPORTUNISTIC_TLS: bool = Field(
-        description="whether to use opportunistic TLS connection to smtp server",
+        description="Enable opportunistic TLS for SMTP connections",
         default=False,
     )
 
 
 class RagEtlConfig(BaseSettings):
     """
-    RAG ETL Configurations.
+    Configuration for RAG ETL processes
     """
 
     ETL_TYPE: str = Field(
-        description="RAG ETL type name, default to `dify`, available values are `dify` and `Unstructured`. ",
+        description="RAG ETL type ('dify' or 'Unstructured'), default to 'dify'",
         default="dify",
     )
 
     KEYWORD_DATA_SOURCE_TYPE: str = Field(
-        description="source type for keyword data, default to `database`, available values are `database` .",
+        description="Data source type for keyword extraction"
+        " ('database' or other supported types), default to 'database'",
         default="database",
     )
 
     UNSTRUCTURED_API_URL: Optional[str] = Field(
-        description="API URL for Unstructured",
+        description="API URL for Unstructured.io service",
         default=None,
     )
 
     UNSTRUCTURED_API_KEY: Optional[str] = Field(
-        description="API key for Unstructured",
+        description="API key for Unstructured.io service",
         default=None,
     )
 
 
 class DataSetConfig(BaseSettings):
     """
-    Dataset configs
+    Configuration for dataset management
     """
 
     CLEAN_DAY_SETTING: PositiveInt = Field(
-        description="interval in days for cleaning up dataset",
+        description="Interval in days for dataset cleanup operations",
         default=30,
     )
 
     DATASET_OPERATOR_ENABLED: bool = Field(
-        description="whether to enable dataset operator",
+        description="Enable or disable dataset operator functionality",
         default=False,
     )
 
 
 class WorkspaceConfig(BaseSettings):
     """
-    Workspace configs
+    Configuration for workspace management
     """
 
     INVITE_EXPIRY_HOURS: PositiveInt = Field(
-        description="workspaces invitation expiration in hours",
+        description="Expiration time in hours for workspace invitation links",
         default=72,
     )
 
 
 class IndexingConfig(BaseSettings):
     """
-    Indexing configs.
+    Configuration for indexing operations
     """
 
     INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: PositiveInt = Field(
-        description="max segmentation token length for indexing",
+        description="Maximum token length for text segmentation during indexing",
         default=1000,
     )
 
 
 class ImageFormatConfig(BaseSettings):
     MULTIMODAL_SEND_IMAGE_FORMAT: str = Field(
-        description="multi model send image format, support base64, url, default is base64",
+        description="Format for sending images in multimodal contexts ('base64' or 'url'), default is base64",
         default="base64",
     )
 
 
 class CeleryBeatConfig(BaseSettings):
     CELERY_BEAT_SCHEDULER_TIME: int = Field(
-        description="the time of the celery scheduler, default to 1 day",
+        description="Interval in days for Celery Beat scheduler execution, default to 1 day",
         default=1,
     )
 
 
 class PositionConfig(BaseSettings):
     POSITION_PROVIDER_PINS: str = Field(
-        description="The heads of model providers",
+        description="Comma-separated list of pinned model providers",
         default="",
     )
 
     POSITION_PROVIDER_INCLUDES: str = Field(
-        description="The included model providers",
+        description="Comma-separated list of included model providers",
         default="",
     )
 
     POSITION_PROVIDER_EXCLUDES: str = Field(
-        description="The excluded model providers",
+        description="Comma-separated list of excluded model providers",
         default="",
     )
 
     POSITION_TOOL_PINS: str = Field(
-        description="The heads of tools",
+        description="Comma-separated list of pinned tools",
         default="",
     )
 
     POSITION_TOOL_INCLUDES: str = Field(
-        description="The included tools",
+        description="Comma-separated list of included tools",
         default="",
     )
 
     POSITION_TOOL_EXCLUDES: str = Field(
-        description="The excluded tools",
+        description="Comma-separated list of excluded tools",
         default="",
     )
 

api/configs/feature/hosted_service/__init__.py

@@ -6,31 +6,31 @@ from pydantic_settings import BaseSettings
 
 class HostedOpenAiConfig(BaseSettings):
     """
-    Hosted OpenAI service config
+    Configuration for hosted OpenAI service
     """
 
     HOSTED_OPENAI_API_KEY: Optional[str] = Field(
-        description="",
+        description="API key for hosted OpenAI service",
         default=None,
     )
 
     HOSTED_OPENAI_API_BASE: Optional[str] = Field(
-        description="",
+        description="Base URL for hosted OpenAI API",
         default=None,
     )
 
     HOSTED_OPENAI_API_ORGANIZATION: Optional[str] = Field(
-        description="",
+        description="Organization ID for hosted OpenAI service",
         default=None,
     )
 
     HOSTED_OPENAI_TRIAL_ENABLED: bool = Field(
-        description="",
+        description="Enable trial access to hosted OpenAI service",
         default=False,
     )
 
     HOSTED_OPENAI_TRIAL_MODELS: str = Field(
-        description="",
+        description="Comma-separated list of available models for trial access",
         default="gpt-3.5-turbo,"
         "gpt-3.5-turbo-1106,"
         "gpt-3.5-turbo-instruct,"
@@ -42,17 +42,17 @@ class HostedOpenAiConfig(BaseSettings):
     )
 
     HOSTED_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field(
-        description="",
+        description="Quota limit for hosted OpenAI service usage",
         default=200,
     )
 
     HOSTED_OPENAI_PAID_ENABLED: bool = Field(
-        description="",
+        description="Enable paid access to hosted OpenAI service",
         default=False,
     )
 
     HOSTED_OPENAI_PAID_MODELS: str = Field(
-        description="",
+        description="Comma-separated list of available models for paid access",
         default="gpt-4,"
         "gpt-4-turbo-preview,"
         "gpt-4-turbo-2024-04-09,"
@@ -71,124 +71,122 @@ class HostedOpenAiConfig(BaseSettings):
 
 class HostedAzureOpenAiConfig(BaseSettings):
     """
-    Hosted OpenAI service config
+    Configuration for hosted Azure OpenAI service
     """
 
     HOSTED_AZURE_OPENAI_ENABLED: bool = Field(
-        description="",
+        description="Enable hosted Azure OpenAI service",
         default=False,
     )
 
     HOSTED_AZURE_OPENAI_API_KEY: Optional[str] = Field(
-        description="",
+        description="API key for hosted Azure OpenAI service",
         default=None,
     )
 
     HOSTED_AZURE_OPENAI_API_BASE: Optional[str] = Field(
-        description="",
+        description="Base URL for hosted Azure OpenAI API",
         default=None,
     )
 
     HOSTED_AZURE_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field(
-        description="",
+        description="Quota limit for hosted Azure OpenAI service usage",
         default=200,
     )
 
 
 class HostedAnthropicConfig(BaseSettings):
     """
-    Hosted Azure OpenAI service config
+    Configuration for hosted Anthropic service
     """
 
     HOSTED_ANTHROPIC_API_BASE: Optional[str] = Field(
-        description="",
+        description="Base URL for hosted Anthropic API",
         default=None,
     )
 
     HOSTED_ANTHROPIC_API_KEY: Optional[str] = Field(
-        description="",
+        description="API key for hosted Anthropic service",
         default=None,
     )
 
     HOSTED_ANTHROPIC_TRIAL_ENABLED: bool = Field(
-        description="",
+        description="Enable trial access to hosted Anthropic service",
         default=False,
     )
 
     HOSTED_ANTHROPIC_QUOTA_LIMIT: NonNegativeInt = Field(
-        description="",
+        description="Quota limit for hosted Anthropic service usage",
         default=600000,
     )
 
     HOSTED_ANTHROPIC_PAID_ENABLED: bool = Field(
-        description="",
+        description="Enable paid access to hosted Anthropic service",
         default=False,
     )
 
 
 class HostedMinmaxConfig(BaseSettings):
     """
-    Hosted Minmax service config
+    Configuration for hosted Minmax service
     """
 
     HOSTED_MINIMAX_ENABLED: bool = Field(
-        description="",
+        description="Enable hosted Minmax service",
         default=False,
     )
 
 
 class HostedSparkConfig(BaseSettings):
     """
-    Hosted Spark service config
+    Configuration for hosted Spark service
     """
 
     HOSTED_SPARK_ENABLED: bool = Field(
-        description="",
+        description="Enable hosted Spark service",
         default=False,
     )
 
 
 class HostedZhipuAIConfig(BaseSettings):
     """
-    Hosted Minmax service config
+    Configuration for hosted ZhipuAI service
     """
 
     HOSTED_ZHIPUAI_ENABLED: bool = Field(
-        description="",
+        description="Enable hosted ZhipuAI service",
         default=False,
     )
 
 
 class HostedModerationConfig(BaseSettings):
     """
-    Hosted Moderation service config
+    Configuration for hosted Moderation service
     """
 
     HOSTED_MODERATION_ENABLED: bool = Field(
-        description="",
+        description="Enable hosted Moderation service",
         default=False,
     )
 
     HOSTED_MODERATION_PROVIDERS: str = Field(
-        description="",
+        description="Comma-separated list of moderation providers",
         default="",
     )
 
 
 class HostedFetchAppTemplateConfig(BaseSettings):
     """
-    Hosted Moderation service config
+    Configuration for fetching app templates
     """
 
     HOSTED_FETCH_APP_TEMPLATES_MODE: str = Field(
-        description="the mode for fetching app templates,"
-        " default to remote,"
-        " available values: remote, db, builtin",
+        description="Mode for fetching app templates: remote, db, or builtin" " default to remote,",
         default="remote",
     )
 
     HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN: str = Field(
-        description="the domain for fetching remote app templates",
+        description="Domain for fetching remote app templates",
         default="https://tmpl.dify.ai",
     )
 

api/configs/middleware/__init__.py

@@ -1,7 +1,7 @@
 from typing import Any, Optional
 from urllib.parse import quote_plus
 
-from pydantic import Field, NonNegativeInt, PositiveInt, computed_field
+from pydantic import Field, NonNegativeInt, PositiveFloat, PositiveInt, computed_field
 from pydantic_settings import BaseSettings
 
 from configs.middleware.cache.redis_config import RedisConfig
@@ -9,8 +9,10 @@ from configs.middleware.storage.aliyun_oss_storage_config import AliyunOSSStorag
 from configs.middleware.storage.amazon_s3_storage_config import S3StorageConfig
 from configs.middleware.storage.azure_blob_storage_config import AzureBlobStorageConfig
 from configs.middleware.storage.google_cloud_storage_config import GoogleCloudStorageConfig
+from configs.middleware.storage.huawei_obs_storage_config import HuaweiCloudOBSStorageConfig
 from configs.middleware.storage.oci_storage_config import OCIStorageConfig
 from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
+from configs.middleware.storage.volcengine_tos_storage_config import VolcengineTOSStorageConfig
 from configs.middleware.vdb.analyticdb_config import AnalyticdbConfig
 from configs.middleware.vdb.chroma_config import ChromaConfig
 from configs.middleware.vdb.elasticsearch_config import ElasticsearchConfig
@@ -29,70 +31,71 @@ from configs.middleware.vdb.weaviate_config import WeaviateConfig
 
 class StorageConfig(BaseSettings):
     STORAGE_TYPE: str = Field(
-        description="storage type,"
-        " default to `local`,"
-        " available values are `local`, `s3`, `azure-blob`, `aliyun-oss`, `google-storage`.",
+        description="Type of storage to use."
+        " Options: 'local', 's3', 'azure-blob', 'aliyun-oss', 'google-storage'. Default is 'local'.",
         default="local",
     )
 
     STORAGE_LOCAL_PATH: str = Field(
-        description="local storage path",
+        description="Path for local storage when STORAGE_TYPE is set to 'local'.",
         default="storage",
     )
 
 
 class VectorStoreConfig(BaseSettings):
     VECTOR_STORE: Optional[str] = Field(
-        description="vector store type",
+        description="Type of vector store to use for efficient similarity search."
+        " Set to None if not using a vector store.",
         default=None,
     )
 
 
 class KeywordStoreConfig(BaseSettings):
     KEYWORD_STORE: str = Field(
-        description="keyword store type",
+        description="Method for keyword extraction and storage."
+        " Default is 'jieba', a Chinese text segmentation library.",
         default="jieba",
     )
 
 
 class DatabaseConfig:
     DB_HOST: str = Field(
-        description="db host",
+        description="Hostname or IP address of the database server.",
         default="localhost",
     )
 
     DB_PORT: PositiveInt = Field(
-        description="db port",
+        description="Port number for database connection.",
         default=5432,
     )
 
     DB_USERNAME: str = Field(
-        description="db username",
+        description="Username for database authentication.",
         default="postgres",
     )
 
     DB_PASSWORD: str = Field(
-        description="db password",
+        description="Password for database authentication.",
         default="",
     )
 
     DB_DATABASE: str = Field(
-        description="db database",
+        description="Name of the database to connect to.",
         default="dify",
     )
 
     DB_CHARSET: str = Field(
-        description="db charset",
+        description="Character set for database connection.",
         default="",
     )
 
     DB_EXTRAS: str = Field(
-        description="db extras options. Example: keepalives_idle=60&keepalives=1",
+        description="Additional database connection parameters. Example: 'keepalives_idle=60&keepalives=1'",
         default="",
     )
 
     SQLALCHEMY_DATABASE_URI_SCHEME: str = Field(
-        description="db uri scheme",
+        description="Database URI scheme for SQLAlchemy connection.",
         default="postgresql",
     )
 
@@ -110,27 +113,27 @@ class DatabaseConfig:
         )
 
     SQLALCHEMY_POOL_SIZE: NonNegativeInt = Field(
-        description="pool size of SqlAlchemy",
+        description="Maximum number of database connections in the pool.",
         default=30,
     )
 
     SQLALCHEMY_MAX_OVERFLOW: NonNegativeInt = Field(
-        description="max overflows for SqlAlchemy",
+        description="Maximum number of connections that can be created beyond the pool_size.",
         default=10,
     )
 
     SQLALCHEMY_POOL_RECYCLE: NonNegativeInt = Field(
-        description="SqlAlchemy pool recycle",
+        description="Number of seconds after which a connection is automatically recycled.",
         default=3600,
     )
 
     SQLALCHEMY_POOL_PRE_PING: bool = Field(
-        description="whether to enable pool pre-ping in SqlAlchemy",
+        description="If True, enables connection pool pre-ping feature to check connections.",
         default=False,
     )
 
     SQLALCHEMY_ECHO: bool | str = Field(
-        description="whether to enable SqlAlchemy echo",
+        description="If True, SQLAlchemy will log all SQL statements.",
         default=False,
     )
 
@@ -148,15 +151,30 @@ class DatabaseConfig:
 
 class CeleryConfig(DatabaseConfig):
     CELERY_BACKEND: str = Field(
-        description="Celery backend, available values are `database`, `redis`",
+        description="Backend for Celery task results. Options: 'database', 'redis'.",
         default="database",
     )
 
     CELERY_BROKER_URL: Optional[str] = Field(
-        description="CELERY_BROKER_URL",
+        description="URL of the message broker for Celery tasks.",
         default=None,
     )
 
+    CELERY_USE_SENTINEL: Optional[bool] = Field(
+        description="Whether to use Redis Sentinel for high availability.",
+        default=False,
+    )
+
+    CELERY_SENTINEL_MASTER_NAME: Optional[str] = Field(
+        description="Name of the Redis Sentinel master.",
+        default=None,
+    )
+
+    CELERY_SENTINEL_SOCKET_TIMEOUT: Optional[PositiveFloat] = Field(
+        description="Timeout for Redis Sentinel socket operations in seconds.",
+        default=0.1,
+    )
+
     @computed_field
     @property
     def CELERY_RESULT_BACKEND(self) -> str | None:
@@ -184,6 +202,8 @@ class MiddlewareConfig(
     AzureBlobStorageConfig,
     GoogleCloudStorageConfig,
     TencentCloudCOSStorageConfig,
+    HuaweiCloudOBSStorageConfig,
+    VolcengineTOSStorageConfig,
     S3StorageConfig,
     OCIStorageConfig,
     # configs of vdb and vdb providers

api/configs/middleware/cache/redis_config.py

@@ -1,40 +1,70 @@
 from typing import Optional
 
-from pydantic import Field, NonNegativeInt, PositiveInt
+from pydantic import Field, NonNegativeInt, PositiveFloat, PositiveInt
 from pydantic_settings import BaseSettings
 
 
 class RedisConfig(BaseSettings):
     """
-    Redis configs
+    Configuration settings for Redis connection
     """
 
     REDIS_HOST: str = Field(
-        description="Redis host",
+        description="Hostname or IP address of the Redis server",
         default="localhost",
     )
 
     REDIS_PORT: PositiveInt = Field(
-        description="Redis port",
+        description="Port number on which the Redis server is listening",
         default=6379,
     )
 
     REDIS_USERNAME: Optional[str] = Field(
-        description="Redis username",
+        description="Username for Redis authentication (if required)",
         default=None,
     )
 
     REDIS_PASSWORD: Optional[str] = Field(
-        description="Redis password",
+        description="Password for Redis authentication (if required)",
         default=None,
     )
 
     REDIS_DB: NonNegativeInt = Field(
-        description="Redis database id, default to 0",
+        description="Redis database number to use (0-15)",
         default=0,
     )
 
     REDIS_USE_SSL: bool = Field(
-        description="whether to use SSL for Redis connection",
+        description="Enable SSL/TLS for the Redis connection",
         default=False,
     )
+
+    REDIS_USE_SENTINEL: Optional[bool] = Field(
+        description="Enable Redis Sentinel mode for high availability",
+        default=False,
+    )
+
+    REDIS_SENTINELS: Optional[str] = Field(
+        description="Comma-separated list of Redis Sentinel nodes (host:port)",
+        default=None,
+    )
+
+    REDIS_SENTINEL_SERVICE_NAME: Optional[str] = Field(
+        description="Name of the Redis Sentinel service to monitor",
+        default=None,
+    )
+
+    REDIS_SENTINEL_USERNAME: Optional[str] = Field(
+        description="Username for Redis Sentinel authentication (if required)",
+        default=None,
+    )
+
+    REDIS_SENTINEL_PASSWORD: Optional[str] = Field(
+        description="Password for Redis Sentinel authentication (if required)",
+        default=None,
+    )
+
+    REDIS_SENTINEL_SOCKET_TIMEOUT: Optional[PositiveFloat] = Field(
+        description="Socket timeout in seconds for Redis Sentinel connections",
+        default=0.1,
+    )

api/configs/middleware/storage/aliyun_oss_storage_config.py

@@ -6,40 +6,40 @@ from pydantic_settings import BaseSettings
 
 class AliyunOSSStorageConfig(BaseSettings):
     """
-    Aliyun storage configs
+    Configuration settings for Aliyun Object Storage Service (OSS)
     """
 
     ALIYUN_OSS_BUCKET_NAME: Optional[str] = Field(
-        description="Aliyun OSS bucket name",
+        description="Name of the Aliyun OSS bucket to store and retrieve objects",
         default=None,
     )
 
     ALIYUN_OSS_ACCESS_KEY: Optional[str] = Field(
-        description="Aliyun OSS access key",
+        description="Access key ID for authenticating with Aliyun OSS",
         default=None,
     )
 
     ALIYUN_OSS_SECRET_KEY: Optional[str] = Field(
-        description="Aliyun OSS secret key",
+        description="Secret access key for authenticating with Aliyun OSS",
         default=None,
     )
 
     ALIYUN_OSS_ENDPOINT: Optional[str] = Field(
-        description="Aliyun OSS endpoint URL",
+        description="URL of the Aliyun OSS endpoint for your chosen region",
         default=None,
     )
 
     ALIYUN_OSS_REGION: Optional[str] = Field(
-        description="Aliyun OSS region",
+        description="Aliyun OSS region where your bucket is located (e.g., 'oss-cn-hangzhou')",
         default=None,
     )
 
     ALIYUN_OSS_AUTH_VERSION: Optional[str] = Field(
-        description="Aliyun OSS authentication version",
+        description="Version of the authentication protocol to use with Aliyun OSS (e.g., 'v4')",
         default=None,
     )
 
     ALIYUN_OSS_PATH: Optional[str] = Field(
-        description="Aliyun OSS path",
+        description="Base path within the bucket to store objects (e.g., 'my-app-data/')",
         default=None,
     )

api/configs/middleware/storage/amazon_s3_storage_config.py

@@ -6,40 +6,40 @@ from pydantic_settings import BaseSettings
 
 class S3StorageConfig(BaseSettings):
     """
-    S3 storage configs
+    Configuration settings for S3-compatible object storage
     """
 
     S3_ENDPOINT: Optional[str] = Field(
-        description="S3 storage endpoint",
+        description="URL of the S3-compatible storage endpoint (e.g., 'https://s3.amazonaws.com')",
         default=None,
     )
 
     S3_REGION: Optional[str] = Field(
-        description="S3 storage region",
+        description="Region where the S3 bucket is located (e.g., 'us-east-1')",
         default=None,
     )
 
     S3_BUCKET_NAME: Optional[str] = Field(
-        description="S3 storage bucket name",
+        description="Name of the S3 bucket to store and retrieve objects",
         default=None,
     )
 
     S3_ACCESS_KEY: Optional[str] = Field(
-        description="S3 storage access key",
+        description="Access key ID for authenticating with the S3 service",
         default=None,
     )
 
     S3_SECRET_KEY: Optional[str] = Field(
-        description="S3 storage secret key",
+        description="Secret access key for authenticating with the S3 service",
         default=None,
     )
 
     S3_ADDRESS_STYLE: str = Field(
-        description="S3 storage address style",
+        description="S3 addressing style: 'auto', 'path', or 'virtual'",
         default="auto",
     )
 
     S3_USE_AWS_MANAGED_IAM: bool = Field(
-        description="whether to use aws managed IAM for S3",
+        description="Use AWS managed IAM roles for authentication instead of access/secret keys",
         default=False,
     )

api/configs/middleware/storage/azure_blob_storage_config.py

@@ -6,25 +6,25 @@ from pydantic_settings import BaseSettings
 
 class AzureBlobStorageConfig(BaseSettings):
     """
-    Azure Blob storage configs
+    Configuration settings for Azure Blob Storage
     """
 
     AZURE_BLOB_ACCOUNT_NAME: Optional[str] = Field(
-        description="Azure Blob account name",
+        description="Name of the Azure Storage account (e.g., 'mystorageaccount')",
         default=None,
     )
 
     AZURE_BLOB_ACCOUNT_KEY: Optional[str] = Field(
-        description="Azure Blob account key",
+        description="Access key for authenticating with the Azure Storage account",
         default=None,
     )
 
     AZURE_BLOB_CONTAINER_NAME: Optional[str] = Field(
-        description="Azure Blob container name",
+        description="Name of the Azure Blob container to store and retrieve objects",
         default=None,
     )
 
     AZURE_BLOB_ACCOUNT_URL: Optional[str] = Field(
-        description="Azure Blob account URL",
+        description="URL of the Azure Blob storage endpoint (e.g., 'https://mystorageaccount.blob.core.windows.net')",
         default=None,
     )

api/configs/middleware/storage/google_cloud_storage_config.py

@@ -6,15 +6,15 @@ from pydantic_settings import BaseSettings
 
 class GoogleCloudStorageConfig(BaseSettings):
     """
-    Google Cloud storage configs
+    Configuration settings for Google Cloud Storage
     """
 
     GOOGLE_STORAGE_BUCKET_NAME: Optional[str] = Field(
-        description="Google Cloud storage bucket name",
+        description="Name of the Google Cloud Storage bucket to store and retrieve objects (e.g., 'my-gcs-bucket')",
         default=None,
     )
 
     GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: Optional[str] = Field(
-        description="Google Cloud storage service account json base64",
+        description="Base64-encoded JSON key file for Google Cloud service account authentication",
         default=None,
     )

api/configs/middleware/storage/huawei_obs_storage_config.py

@@ -0,0 +1,29 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class HuaweiCloudOBSStorageConfig(BaseModel):
+    """
+    Configuration settings for Huawei Cloud Object Storage Service (OBS)
+    """
+
+    HUAWEI_OBS_BUCKET_NAME: Optional[str] = Field(
+        description="Name of the Huawei Cloud OBS bucket to store and retrieve objects (e.g., 'my-obs-bucket')",
+        default=None,
+    )
+
+    HUAWEI_OBS_ACCESS_KEY: Optional[str] = Field(
+        description="Access Key ID for authenticating with Huawei Cloud OBS",
+        default=None,
+    )
+
+    HUAWEI_OBS_SECRET_KEY: Optional[str] = Field(
+        description="Secret Access Key for authenticating with Huawei Cloud OBS",
+        default=None,
+    )
+
+    HUAWEI_OBS_SERVER: Optional[str] = Field(
+        description="Endpoint URL for Huawei Cloud OBS (e.g., 'https://obs.cn-north-4.myhuaweicloud.com')",
+        default=None,
+    )

api/configs/middleware/storage/oci_storage_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class OCIStorageConfig(BaseSettings):
     """
-    OCI storage configs
+    Configuration settings for Oracle Cloud Infrastructure (OCI) Object Storage
     """
 
     OCI_ENDPOINT: Optional[str] = Field(
-        description="OCI storage endpoint",
+        description="URL of the OCI Object Storage endpoint (e.g., 'https://objectstorage.us-phoenix-1.oraclecloud.com')",
         default=None,
     )
 
     OCI_REGION: Optional[str] = Field(
-        description="OCI storage region",
+        description="OCI region where the bucket is located (e.g., 'us-phoenix-1')",
         default=None,
     )
 
     OCI_BUCKET_NAME: Optional[str] = Field(
-        description="OCI storage bucket name",
+        description="Name of the OCI Object Storage bucket to store and retrieve objects (e.g., 'my-oci-bucket')",
         default=None,
     )
 
     OCI_ACCESS_KEY: Optional[str] = Field(
-        description="OCI storage access key",
+        description="Access key (also known as API key) for authenticating with OCI Object Storage",
         default=None,
     )
 
     OCI_SECRET_KEY: Optional[str] = Field(
-        description="OCI storage secret key",
+        description="Secret key associated with the access key for authenticating with OCI Object Storage",
         default=None,
     )

api/configs/middleware/storage/tencent_cos_storage_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class TencentCloudCOSStorageConfig(BaseSettings):
     """
-    Tencent Cloud COS storage configs
+    Configuration settings for Tencent Cloud Object Storage (COS)
     """
 
     TENCENT_COS_BUCKET_NAME: Optional[str] = Field(
-        description="Tencent Cloud COS bucket name",
+        description="Name of the Tencent Cloud COS bucket to store and retrieve objects",
         default=None,
     )
 
     TENCENT_COS_REGION: Optional[str] = Field(
-        description="Tencent Cloud COS region",
+        description="Tencent Cloud region where the COS bucket is located (e.g., 'ap-guangzhou')",
         default=None,
     )
 
     TENCENT_COS_SECRET_ID: Optional[str] = Field(
-        description="Tencent Cloud COS secret id",
+        description="SecretId for authenticating with Tencent Cloud COS (part of API credentials)",
         default=None,
     )
 
     TENCENT_COS_SECRET_KEY: Optional[str] = Field(
-        description="Tencent Cloud COS secret key",
+        description="SecretKey for authenticating with Tencent Cloud COS (part of API credentials)",
         default=None,
     )
 
     TENCENT_COS_SCHEME: Optional[str] = Field(
-        description="Tencent Cloud COS scheme",
+        description="Protocol scheme for COS requests: 'https' (recommended) or 'http'",
         default=None,
     )

api/configs/middleware/storage/volcengine_tos_storage_config.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class VolcengineTOSStorageConfig(BaseModel):
+    """
+    Configuration settings for Volcengine Tinder Object Storage (TOS)
+    """
+
+    VOLCENGINE_TOS_BUCKET_NAME: Optional[str] = Field(
+        description="Name of the Volcengine TOS bucket to store and retrieve objects (e.g., 'my-tos-bucket')",
+        default=None,
+    )
+
+    VOLCENGINE_TOS_ACCESS_KEY: Optional[str] = Field(
+        description="Access Key ID for authenticating with Volcengine TOS",
+        default=None,
+    )
+
+    VOLCENGINE_TOS_SECRET_KEY: Optional[str] = Field(
+        description="Secret Access Key for authenticating with Volcengine TOS",
+        default=None,
+    )
+
+    VOLCENGINE_TOS_ENDPOINT: Optional[str] = Field(
+        description="URL of the Volcengine TOS endpoint (e.g., 'https://tos-cn-beijing.volces.com')",
+        default=None,
+    )
+
+    VOLCENGINE_TOS_REGION: Optional[str] = Field(
+        description="Volcengine region where the TOS bucket is located (e.g., 'cn-beijing')",
+        default=None,
+    )

api/configs/middleware/vdb/analyticdb_config.py

@@ -5,33 +5,38 @@ from pydantic import BaseModel, Field
 
 class AnalyticdbConfig(BaseModel):
     """
-    Configuration for connecting to AnalyticDB.
+    Configuration for connecting to Alibaba Cloud AnalyticDB for PostgreSQL.
     Refer to the following documentation for details on obtaining credentials:
     https://www.alibabacloud.com/help/en/analyticdb-for-postgresql/getting-started/create-an-instance-instances-with-vector-engine-optimization-enabled
     """
 
     ANALYTICDB_KEY_ID: Optional[str] = Field(
-        default=None, description="The Access Key ID provided by Alibaba Cloud for authentication."
+        default=None, description="The Access Key ID provided by Alibaba Cloud for API authentication."
     )
     ANALYTICDB_KEY_SECRET: Optional[str] = Field(
-        default=None, description="The Secret Access Key corresponding to the Access Key ID for secure access."
+        default=None, description="The Secret Access Key corresponding to the Access Key ID for secure API access."
     )
     ANALYTICDB_REGION_ID: Optional[str] = Field(
-        default=None, description="The region where the AnalyticDB instance is deployed (e.g., 'cn-hangzhou')."
+        default=None,
+        description="The region where the AnalyticDB instance is deployed (e.g., 'cn-hangzhou', 'ap-southeast-1').",
     )
     ANALYTICDB_INSTANCE_ID: Optional[str] = Field(
         default=None,
-        description="The unique identifier of the AnalyticDB instance you want to connect to (e.g., 'gp-ab123456')..",
+        description="The unique identifier of the AnalyticDB instance you want to connect to.",
     )
     ANALYTICDB_ACCOUNT: Optional[str] = Field(
-        default=None, description="The account name used to log in to the AnalyticDB instance."
+        default=None,
+        description="The account name used to log in to the AnalyticDB instance"
+        " (usually the initial account created with the instance).",
     )
     ANALYTICDB_PASSWORD: Optional[str] = Field(
-        default=None, description="The password associated with the AnalyticDB account for authentication."
+        default=None, description="The password associated with the AnalyticDB account for database authentication."
     )
     ANALYTICDB_NAMESPACE: Optional[str] = Field(
-        default=None, description="The namespace within AnalyticDB for schema isolation."
+        default=None, description="The namespace within AnalyticDB for schema isolation (if using namespace feature)."
     )
     ANALYTICDB_NAMESPACE_PASSWORD: Optional[str] = Field(
-        default=None, description="The password for accessing the specified namespace within the AnalyticDB instance."
+        default=None,
+        description="The password for accessing the specified namespace within the AnalyticDB instance"
+        " (if namespace feature is enabled).",
     )

api/configs/middleware/vdb/chroma_config.py

@@ -6,35 +6,35 @@ from pydantic_settings import BaseSettings
 
 class ChromaConfig(BaseSettings):
     """
-    Chroma configs
+    Configuration settings for Chroma vector database
     """
 
     CHROMA_HOST: Optional[str] = Field(
-        description="Chroma host",
+        description="Hostname or IP address of the Chroma server (e.g., 'localhost' or '192.168.1.100')",
         default=None,
     )
 
     CHROMA_PORT: PositiveInt = Field(
-        description="Chroma port",
+        description="Port number on which the Chroma server is listening (default is 8000)",
         default=8000,
     )
 
     CHROMA_TENANT: Optional[str] = Field(
-        description="Chroma database",
+        description="Tenant identifier for multi-tenancy support in Chroma",
         default=None,
     )
 
     CHROMA_DATABASE: Optional[str] = Field(
-        description="Chroma database",
+        description="Name of the Chroma database to connect to",
         default=None,
     )
 
     CHROMA_AUTH_PROVIDER: Optional[str] = Field(
-        description="Chroma authentication provider",
+        description="Authentication provider for Chroma (e.g., 'basic', 'token', or a custom provider)",
         default=None,
     )
 
     CHROMA_AUTH_CREDENTIALS: Optional[str] = Field(
-        description="Chroma authentication credentials",
+        description="Authentication credentials for Chroma (format depends on the auth provider)",
         default=None,
     )

api/configs/middleware/vdb/elasticsearch_config.py

@@ -6,25 +6,25 @@ from pydantic_settings import BaseSettings
 
 class ElasticsearchConfig(BaseSettings):
     """
-    Elasticsearch configs
+    Configuration settings for Elasticsearch
     """
 
     ELASTICSEARCH_HOST: Optional[str] = Field(
-        description="Elasticsearch host",
+        description="Hostname or IP address of the Elasticsearch server (e.g., 'localhost' or '192.168.1.100')",
         default="127.0.0.1",
     )
 
     ELASTICSEARCH_PORT: PositiveInt = Field(
-        description="Elasticsearch port",
+        description="Port number on which the Elasticsearch server is listening (default is 9200)",
         default=9200,
     )
 
     ELASTICSEARCH_USERNAME: Optional[str] = Field(
-        description="Elasticsearch username",
+        description="Username for authenticating with Elasticsearch (default is 'elastic')",
         default="elastic",
     )
 
     ELASTICSEARCH_PASSWORD: Optional[str] = Field(
-        description="Elasticsearch password",
+        description="Password for authenticating with Elasticsearch (default is 'elastic')",
         default="elastic",
     )

api/configs/middleware/vdb/milvus_config.py

@@ -1,40 +1,35 @@
 from typing import Optional
 
-from pydantic import Field, PositiveInt
+from pydantic import Field
 from pydantic_settings import BaseSettings
 
 
 class MilvusConfig(BaseSettings):
     """
-    Milvus configs
+    Configuration settings for Milvus vector database
     """
 
-    MILVUS_HOST: Optional[str] = Field(
-        description="Milvus host",
+    MILVUS_URI: Optional[str] = Field(
+        description="URI for connecting to the Milvus server (e.g., 'http://localhost:19530' or 'https://milvus-instance.example.com:19530')",
+        default="http://127.0.0.1:19530",
+    )
+
+    MILVUS_TOKEN: Optional[str] = Field(
+        description="Authentication token for Milvus, if token-based authentication is enabled",
         default=None,
     )
 
-    MILVUS_PORT: PositiveInt = Field(
-        description="Milvus RestFul API port",
-        default=9091,
-    )
-
     MILVUS_USER: Optional[str] = Field(
-        description="Milvus user",
+        description="Username for authenticating with Milvus, if username/password authentication is enabled",
         default=None,
     )
 
     MILVUS_PASSWORD: Optional[str] = Field(
-        description="Milvus password",
+        description="Password for authenticating with Milvus, if username/password authentication is enabled",
         default=None,
     )
 
-    MILVUS_SECURE: bool = Field(
-        description="whether to use SSL connection for Milvus",
-        default=False,
-    )
-
     MILVUS_DATABASE: str = Field(
-        description="Milvus database, default to `default`",
+        description="Name of the Milvus database to connect to (default is 'default')",
         default="default",
     )

api/configs/middleware/vdb/myscale_config.py

@@ -3,35 +3,35 @@ from pydantic import BaseModel, Field, PositiveInt
 
 class MyScaleConfig(BaseModel):
     """
-    MyScale configs
+    Configuration settings for MyScale vector database
     """
 
     MYSCALE_HOST: str = Field(
-        description="MyScale host",
+        description="Hostname or IP address of the MyScale server (e.g., 'localhost' or 'myscale.example.com')",
         default="localhost",
     )
 
     MYSCALE_PORT: PositiveInt = Field(
-        description="MyScale port",
+        description="Port number on which the MyScale server is listening (default is 8123)",
         default=8123,
     )
 
     MYSCALE_USER: str = Field(
-        description="MyScale user",
+        description="Username for authenticating with MyScale (default is 'default')",
         default="default",
     )
 
     MYSCALE_PASSWORD: str = Field(
-        description="MyScale password",
+        description="Password for authenticating with MyScale (default is an empty string)",
         default="",
     )
 
     MYSCALE_DATABASE: str = Field(
-        description="MyScale database name",
+        description="Name of the MyScale database to connect to (default is 'default')",
         default="default",
     )
 
     MYSCALE_FTS_PARAMS: str = Field(
-        description="MyScale fts index parameters",
+        description="Additional parameters for MyScale Full Text Search index)",
         default="",
     )

api/configs/middleware/vdb/opensearch_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class OpenSearchConfig(BaseSettings):
     """
-    OpenSearch configs
+    Configuration settings for OpenSearch
     """
 
     OPENSEARCH_HOST: Optional[str] = Field(
-        description="OpenSearch host",
+        description="Hostname or IP address of the OpenSearch server (e.g., 'localhost' or 'opensearch.example.com')",
         default=None,
     )
 
     OPENSEARCH_PORT: PositiveInt = Field(
-        description="OpenSearch port",
+        description="Port number on which the OpenSearch server is listening (default is 9200)",
         default=9200,
     )
 
     OPENSEARCH_USER: Optional[str] = Field(
-        description="OpenSearch user",
+        description="Username for authenticating with OpenSearch",
         default=None,
     )
 
     OPENSEARCH_PASSWORD: Optional[str] = Field(
-        description="OpenSearch password",
+        description="Password for authenticating with OpenSearch",
         default=None,
     )
 
     OPENSEARCH_SECURE: bool = Field(
-        description="whether to use SSL connection for OpenSearch",
+        description="Whether to use SSL/TLS encrypted connection for OpenSearch (True for HTTPS, False for HTTP)",
         default=False,
     )

api/configs/middleware/vdb/oracle_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class OracleConfig(BaseSettings):
     """
-    ORACLE configs
+    Configuration settings for Oracle database
     """
 
     ORACLE_HOST: Optional[str] = Field(
-        description="ORACLE host",
+        description="Hostname or IP address of the Oracle database server (e.g., 'localhost' or 'oracle.example.com')",
         default=None,
     )
 
     ORACLE_PORT: Optional[PositiveInt] = Field(
-        description="ORACLE port",
+        description="Port number on which the Oracle database server is listening (default is 1521)",
         default=1521,
     )
 
     ORACLE_USER: Optional[str] = Field(
-        description="ORACLE user",
+        description="Username for authenticating with the Oracle database",
         default=None,
     )
 
     ORACLE_PASSWORD: Optional[str] = Field(
-        description="ORACLE password",
+        description="Password for authenticating with the Oracle database",
         default=None,
     )
 
     ORACLE_DATABASE: Optional[str] = Field(
-        description="ORACLE database",
+        description="Name of the Oracle database or service to connect to (e.g., 'ORCL' or 'pdborcl')",
         default=None,
     )

api/configs/middleware/vdb/pgvector_config.py

@@ -6,30 +6,40 @@ from pydantic_settings import BaseSettings
 
 class PGVectorConfig(BaseSettings):
     """
-    PGVector configs
+    Configuration settings for PGVector (PostgreSQL with vector extension)
     """
 
     PGVECTOR_HOST: Optional[str] = Field(
-        description="PGVector host",
+        description="Hostname or IP address of the PostgreSQL server with PGVector extension (e.g., 'localhost')",
         default=None,
     )
 
     PGVECTOR_PORT: Optional[PositiveInt] = Field(
-        description="PGVector port",
+        description="Port number on which the PostgreSQL server is listening (default is 5433)",
         default=5433,
     )
 
     PGVECTOR_USER: Optional[str] = Field(
-        description="PGVector user",
+        description="Username for authenticating with the PostgreSQL database",
         default=None,
     )
 
     PGVECTOR_PASSWORD: Optional[str] = Field(
-        description="PGVector password",
+        description="Password for authenticating with the PostgreSQL database",
         default=None,
     )
 
     PGVECTOR_DATABASE: Optional[str] = Field(
-        description="PGVector database",
+        description="Name of the PostgreSQL database to connect to",
         default=None,
     )
+
+    PGVECTOR_MIN_CONNECTION: PositiveInt = Field(
+        description="Min connection of the PostgreSQL database",
+        default=1,
+    )
+
+    PGVECTOR_MAX_CONNECTION: PositiveInt = Field(
+        description="Max connection of the PostgreSQL database",
+        default=5,
+    )

api/configs/middleware/vdb/pgvectors_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class PGVectoRSConfig(BaseSettings):
     """
-    PGVectoRS configs
+    Configuration settings for PGVecto.RS (Rust-based vector extension for PostgreSQL)
     """
 
     PGVECTO_RS_HOST: Optional[str] = Field(
-        description="PGVectoRS host",
+        description="Hostname or IP address of the PostgreSQL server with PGVecto.RS extension (e.g., 'localhost')",
         default=None,
     )
 
     PGVECTO_RS_PORT: Optional[PositiveInt] = Field(
-        description="PGVectoRS port",
+        description="Port number on which the PostgreSQL server with PGVecto.RS is listening (default is 5431)",
         default=5431,
     )
 
     PGVECTO_RS_USER: Optional[str] = Field(
-        description="PGVectoRS user",
+        description="Username for authenticating with the PostgreSQL database using PGVecto.RS",
         default=None,
     )
 
     PGVECTO_RS_PASSWORD: Optional[str] = Field(
-        description="PGVectoRS password",
+        description="Password for authenticating with the PostgreSQL database using PGVecto.RS",
         default=None,
     )
 
     PGVECTO_RS_DATABASE: Optional[str] = Field(
-        description="PGVectoRS database",
+        description="Name of the PostgreSQL database with PGVecto.RS extension to connect to",
         default=None,
     )

api/configs/middleware/vdb/qdrant_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class QdrantConfig(BaseSettings):
     """
-    Qdrant configs
+    Configuration settings for Qdrant vector database
     """
 
     QDRANT_URL: Optional[str] = Field(
-        description="Qdrant url",
+        description="URL of the Qdrant server (e.g., 'http://localhost:6333' or 'https://qdrant.example.com')",
         default=None,
     )
 
     QDRANT_API_KEY: Optional[str] = Field(
-        description="Qdrant api key",
+        description="API key for authenticating with the Qdrant server",
         default=None,
     )
 
     QDRANT_CLIENT_TIMEOUT: NonNegativeInt = Field(
-        description="Qdrant client timeout in seconds",
+        description="Timeout in seconds for Qdrant client operations (default is 20 seconds)",
         default=20,
     )
 
     QDRANT_GRPC_ENABLED: bool = Field(
-        description="whether enable grpc support for Qdrant connection",
+        description="Whether to enable gRPC support for Qdrant connection (True for gRPC, False for HTTP)",
         default=False,
     )
 
     QDRANT_GRPC_PORT: PositiveInt = Field(
-        description="Qdrant grpc port",
+        description="Port number for gRPC connection to Qdrant server (default is 6334)",
         default=6334,
     )

api/configs/middleware/vdb/relyt_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class RelytConfig(BaseSettings):
     """
-    Relyt configs
+    Configuration settings for Relyt database
     """
 
     RELYT_HOST: Optional[str] = Field(
-        description="Relyt host",
+        description="Hostname or IP address of the Relyt server (e.g., 'localhost' or 'relyt.example.com')",
         default=None,
     )
 
     RELYT_PORT: PositiveInt = Field(
-        description="Relyt port",
+        description="Port number on which the Relyt server is listening (default is 9200)",
         default=9200,
     )
 
     RELYT_USER: Optional[str] = Field(
-        description="Relyt user",
+        description="Username for authenticating with the Relyt database",
         default=None,
     )
 
     RELYT_PASSWORD: Optional[str] = Field(
-        description="Relyt password",
+        description="Password for authenticating with the Relyt database",
         default=None,
     )
 
     RELYT_DATABASE: Optional[str] = Field(
-        description="Relyt database",
+        description="Name of the Relyt database to connect to (default is 'default')",
         default="default",
     )

api/configs/middleware/vdb/tencent_vector_config.py

@@ -6,45 +6,45 @@ from pydantic_settings import BaseSettings
 
 class TencentVectorDBConfig(BaseSettings):
     """
-    Tencent Vector configs
+    Configuration settings for Tencent Vector Database
     """
 
     TENCENT_VECTOR_DB_URL: Optional[str] = Field(
-        description="Tencent Vector URL",
+        description="URL of the Tencent Vector Database service (e.g., 'https://vectordb.tencentcloudapi.com')",
         default=None,
     )
 
     TENCENT_VECTOR_DB_API_KEY: Optional[str] = Field(
-        description="Tencent Vector API key",
+        description="API key for authenticating with the Tencent Vector Database service",
         default=None,
     )
 
     TENCENT_VECTOR_DB_TIMEOUT: PositiveInt = Field(
-        description="Tencent Vector timeout in seconds",
+        description="Timeout in seconds for Tencent Vector Database operations (default is 30 seconds)",
         default=30,
     )
 
     TENCENT_VECTOR_DB_USERNAME: Optional[str] = Field(
-        description="Tencent Vector username",
+        description="Username for authenticating with the Tencent Vector Database (if required)",
         default=None,
     )
 
     TENCENT_VECTOR_DB_PASSWORD: Optional[str] = Field(
-        description="Tencent Vector password",
+        description="Password for authenticating with the Tencent Vector Database (if required)",
         default=None,
     )
 
     TENCENT_VECTOR_DB_SHARD: PositiveInt = Field(
-        description="Tencent Vector sharding number",
+        description="Number of shards for the Tencent Vector Database (default is 1)",
         default=1,
     )
 
     TENCENT_VECTOR_DB_REPLICAS: NonNegativeInt = Field(
-        description="Tencent Vector replicas",
+        description="Number of replicas for the Tencent Vector Database (default is 2)",
         default=2,
     )
 
     TENCENT_VECTOR_DB_DATABASE: Optional[str] = Field(
-        description="Tencent Vector Database",
+        description="Name of the specific Tencent Vector Database to connect to",
         default=None,
     )

api/configs/middleware/vdb/tidb_vector_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class TiDBVectorConfig(BaseSettings):
     """
-    TiDB Vector configs
+    Configuration settings for TiDB Vector database
     """
 
     TIDB_VECTOR_HOST: Optional[str] = Field(
-        description="TiDB Vector host",
+        description="Hostname or IP address of the TiDB Vector server (e.g., 'localhost' or 'tidb.example.com')",
         default=None,
     )
 
     TIDB_VECTOR_PORT: Optional[PositiveInt] = Field(
-        description="TiDB Vector port",
+        description="Port number on which the TiDB Vector server is listening (default is 4000)",
         default=4000,
     )
 
     TIDB_VECTOR_USER: Optional[str] = Field(
-        description="TiDB Vector user",
+        description="Username for authenticating with the TiDB Vector database",
         default=None,
     )
 
     TIDB_VECTOR_PASSWORD: Optional[str] = Field(
-        description="TiDB Vector password",
+        description="Password for authenticating with the TiDB Vector database",
         default=None,
     )
 
     TIDB_VECTOR_DATABASE: Optional[str] = Field(
-        description="TiDB Vector database",
+        description="Name of the TiDB Vector database to connect to",
         default=None,
     )

api/configs/middleware/vdb/weaviate_config.py

@@ -6,25 +6,25 @@ from pydantic_settings import BaseSettings
 
 class WeaviateConfig(BaseSettings):
     """
-    Weaviate configs
+    Configuration settings for Weaviate vector database
     """
 
     WEAVIATE_ENDPOINT: Optional[str] = Field(
-        description="Weaviate endpoint URL",
+        description="URL of the Weaviate server (e.g., 'http://localhost:8080' or 'https://weaviate.example.com')",
         default=None,
     )
 
     WEAVIATE_API_KEY: Optional[str] = Field(
-        description="Weaviate API key",
+        description="API key for authenticating with the Weaviate server",
         default=None,
     )
 
     WEAVIATE_GRPC_ENABLED: bool = Field(
-        description="whether to enable gRPC for Weaviate connection",
+        description="Whether to enable gRPC for Weaviate connection (True for gRPC, False for HTTP)",
         default=True,
     )
 
     WEAVIATE_BATCH_SIZE: PositiveInt = Field(
-        description="Weaviate batch size",
+        description="Number of objects to be processed in a single batch operation (default is 100)",
         default=100,
     )

api/configs/packaging/__init__.py

@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
 
     CURRENT_VERSION: str = Field(
         description="Dify version",
-        default="0.7.3",
+        default="0.9.0",
     )
 
     COMMIT_SHA: str = Field(

api/constants/__init__.py

@@ -1 +1,2 @@
 HIDDEN_VALUE = "[__HIDDEN__]"
+UUID_NIL = "00000000-0000-0000-0000-000000000000"

api/controllers/console/__init__.py

@@ -37,7 +37,16 @@ from .auth import activate, data_source_bearer_auth, data_source_oauth, forgot_p
 from .billing import billing
 
 # Import datasets controllers
-from .datasets import data_source, datasets, datasets_document, datasets_segments, file, hit_testing, website
+from .datasets import (
+    data_source,
+    datasets,
+    datasets_document,
+    datasets_segments,
+    external,
+    file,
+    hit_testing,
+    website,
+)
 
 # Import explore controllers
 from .explore import (

api/controllers/console/admin.py

@@ -60,23 +60,15 @@ class InsertExploreAppListApi(Resource):
 
         site = app.site
         if not site:
-            desc = args["desc"] if args["desc"] else ""
-            copy_right = args["copyright"] if args["copyright"] else ""
-            privacy_policy = args["privacy_policy"] if args["privacy_policy"] else ""
-            custom_disclaimer = args["custom_disclaimer"] if args["custom_disclaimer"] else ""
+            desc = args["desc"] or ""
+            copy_right = args["copyright"] or ""
+            privacy_policy = args["privacy_policy"] or ""
+            custom_disclaimer = args["custom_disclaimer"] or ""
         else:
-            desc = site.description if site.description else args["desc"] if args["desc"] else ""
-            copy_right = site.copyright if site.copyright else args["copyright"] if args["copyright"] else ""
-            privacy_policy = (
-                site.privacy_policy if site.privacy_policy else args["privacy_policy"] if args["privacy_policy"] else ""
-            )
-            custom_disclaimer = (
-                site.custom_disclaimer
-                if site.custom_disclaimer
-                else args["custom_disclaimer"]
-                if args["custom_disclaimer"]
-                else ""
-            )
+            desc = site.description or args["desc"] or ""
+            copy_right = site.copyright or args["copyright"] or ""
+            privacy_policy = site.privacy_policy or args["privacy_policy"] or ""
+            custom_disclaimer = site.custom_disclaimer or args["custom_disclaimer"] or ""
 
         recommended_app = RecommendedApp.query.filter(RecommendedApp.app_id == args["app_id"]).first()
 

api/controllers/console/apikey.py

@@ -57,7 +57,7 @@ class BaseApiKeyListResource(Resource):
     def post(self, resource_id):
         resource_id = str(resource_id)
         _get_resource(resource_id, current_user.current_tenant_id, self.resource_model)
-        if not current_user.is_admin_or_owner:
+        if not current_user.is_editor:
             raise Forbidden()
 
         current_key_count = (

api/controllers/console/app/audio.py

@@ -94,19 +94,15 @@ class ChatMessageTextApi(Resource):
             message_id = args.get("message_id", None)
             text = args.get("text", None)
             if (
-                app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]
+                app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}
                 and app_model.workflow
                 and app_model.workflow.features_dict
             ):
                 text_to_speech = app_model.workflow.features_dict.get("text_to_speech")
-                voice = args.get("voice") if args.get("voice") else text_to_speech.get("voice")
+                voice = args.get("voice") or text_to_speech.get("voice")
             else:
                 try:
-                    voice = (
-                        args.get("voice")
-                        if args.get("voice")
-                        else app_model.app_model_config.text_to_speech_dict.get("voice")
-                    )
+                    voice = args.get("voice") or app_model.app_model_config.text_to_speech_dict.get("voice")
                 except Exception:
                     voice = None
             response = AudioService.transcript_tts(app_model=app_model, text=text, message_id=message_id, voice=voice)

api/controllers/console/app/completion.py

@@ -109,6 +109,7 @@ class ChatMessageApi(Resource):
         parser.add_argument("files", type=list, required=False, location="json")
         parser.add_argument("model_config", type=dict, required=True, location="json")
         parser.add_argument("conversation_id", type=uuid_value, location="json")
+        parser.add_argument("parent_message_id", type=uuid_value, required=False, location="json")
         parser.add_argument("response_mode", type=str, choices=["blocking", "streaming"], location="json")
         parser.add_argument("retriever_from", type=str, required=False, default="dev", location="json")
         args = parser.parse_args()

api/controllers/console/app/conversation.py

@@ -20,7 +20,7 @@ from fields.conversation_fields import (
     conversation_pagination_fields,
     conversation_with_summary_pagination_fields,
 )
-from libs.helper import datetime_string
+from libs.helper import DatetimeString
 from libs.login import login_required
 from models.model import AppMode, Conversation, EndUser, Message, MessageAnnotation
 
@@ -36,8 +36,8 @@ class CompletionConversationApi(Resource):
             raise Forbidden()
         parser = reqparse.RequestParser()
         parser.add_argument("keyword", type=str, location="args")
-        parser.add_argument("start", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
-        parser.add_argument("end", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("start", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
         parser.add_argument(
             "annotation_status", type=str, choices=["annotated", "not_annotated", "all"], default="all", location="args"
         )
@@ -143,8 +143,8 @@ class ChatConversationApi(Resource):
             raise Forbidden()
         parser = reqparse.RequestParser()
         parser.add_argument("keyword", type=str, location="args")
-        parser.add_argument("start", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
-        parser.add_argument("end", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("start", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
         parser.add_argument(
             "annotation_status", type=str, choices=["annotated", "not_annotated", "all"], default="all", location="args"
         )
@@ -173,18 +173,21 @@ class ChatConversationApi(Resource):
 
         if args["keyword"]:
             keyword_filter = "%{}%".format(args["keyword"])
-            message_subquery = (
-                db.session.query(Message.conversation_id)
-                .filter(or_(Message.query.ilike(keyword_filter), Message.answer.ilike(keyword_filter)))
-                .subquery()
-            )
-            query = query.join(subquery, subquery.c.conversation_id == Conversation.id).filter(
-                or_(
-                    Conversation.id.in_(message_subquery),
-                    Conversation.name.ilike(keyword_filter),
-                    Conversation.introduction.ilike(keyword_filter),
-                    subquery.c.from_end_user_session_id.ilike(keyword_filter),
-                ),
+            query = (
+                query.join(
+                    Message,
+                    Message.conversation_id == Conversation.id,
+                )
+                .join(subquery, subquery.c.conversation_id == Conversation.id)
+                .filter(
+                    or_(
+                        Message.query.ilike(keyword_filter),
+                        Message.answer.ilike(keyword_filter),
+                        Conversation.name.ilike(keyword_filter),
+                        Conversation.introduction.ilike(keyword_filter),
+                        subquery.c.from_end_user_session_id.ilike(keyword_filter),
+                    ),
+                )
             )
 
         account = current_user
@@ -198,7 +201,11 @@ class ChatConversationApi(Resource):
             start_datetime_timezone = timezone.localize(start_datetime)
             start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
 
-            query = query.where(Conversation.created_at >= start_datetime_utc)
+            match args["sort_by"]:
+                case "updated_at" | "-updated_at":
+                    query = query.where(Conversation.updated_at >= start_datetime_utc)
+                case "created_at" | "-created_at" | _:
+                    query = query.where(Conversation.created_at >= start_datetime_utc)
 
         if args["end"]:
             end_datetime = datetime.strptime(args["end"], "%Y-%m-%d %H:%M")
@@ -207,7 +214,11 @@ class ChatConversationApi(Resource):
             end_datetime_timezone = timezone.localize(end_datetime)
             end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
 
-            query = query.where(Conversation.created_at < end_datetime_utc)
+            match args["sort_by"]:
+                case "updated_at" | "-updated_at":
+                    query = query.where(Conversation.updated_at <= end_datetime_utc)
+                case "created_at" | "-created_at" | _:
+                    query = query.where(Conversation.created_at <= end_datetime_utc)
 
         if args["annotation_status"] == "annotated":
             query = query.options(joinedload(Conversation.message_annotations)).join(

api/controllers/console/app/message.py

@@ -105,8 +105,6 @@ class ChatMessageListApi(Resource):
             if rest_count > 0:
                 has_more = True
 
-        history_messages = list(reversed(history_messages))
-
         return InfiniteScrollPagination(data=history_messages, limit=args["limit"], has_more=has_more)
 
 

api/controllers/console/app/statistic.py

@@ -11,7 +11,7 @@ from controllers.console.app.wraps import get_app_model
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
-from libs.helper import datetime_string
+from libs.helper import DatetimeString
 from libs.login import login_required
 from models.model import AppMode
 
@@ -25,14 +25,17 @@ class DailyMessageStatistic(Resource):
         account = current_user
 
         parser = reqparse.RequestParser()
-        parser.add_argument("start", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
-        parser.add_argument("end", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("start", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
         args = parser.parse_args()
 
-        sql_query = """
-        SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, count(*) AS message_count
-            FROM messages where app_id = :app_id 
-        """
+        sql_query = """SELECT
+    DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date,
+    COUNT(*) AS message_count
+FROM
+    messages
+WHERE
+    app_id = :app_id"""
         arg_dict = {"tz": account.timezone, "app_id": app_model.id}
 
         timezone = pytz.timezone(account.timezone)
@@ -45,7 +48,7 @@ class DailyMessageStatistic(Resource):
             start_datetime_timezone = timezone.localize(start_datetime)
             start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at >= :start"
+            sql_query += " AND created_at >= :start"
             arg_dict["start"] = start_datetime_utc
 
         if args["end"]:
@@ -55,10 +58,10 @@ class DailyMessageStatistic(Resource):
             end_datetime_timezone = timezone.localize(end_datetime)
             end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at < :end"
+            sql_query += " AND created_at < :end"
             arg_dict["end"] = end_datetime_utc
 
-        sql_query += " GROUP BY date order by date"
+        sql_query += " GROUP BY date ORDER BY date"
 
         response_data = []
 
@@ -79,14 +82,17 @@ class DailyConversationStatistic(Resource):
         account = current_user
 
         parser = reqparse.RequestParser()
-        parser.add_argument("start", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
-        parser.add_argument("end", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("start", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
         args = parser.parse_args()
 
-        sql_query = """
-        SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, count(distinct messages.conversation_id) AS conversation_count
-            FROM messages where app_id = :app_id 
-        """
+        sql_query = """SELECT
+    DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date,
+    COUNT(DISTINCT messages.conversation_id) AS conversation_count
+FROM
+    messages
+WHERE
+    app_id = :app_id"""
         arg_dict = {"tz": account.timezone, "app_id": app_model.id}
 
         timezone = pytz.timezone(account.timezone)
@@ -99,7 +105,7 @@ class DailyConversationStatistic(Resource):
             start_datetime_timezone = timezone.localize(start_datetime)
             start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at >= :start"
+            sql_query += " AND created_at >= :start"
             arg_dict["start"] = start_datetime_utc
 
         if args["end"]:
@@ -109,10 +115,10 @@ class DailyConversationStatistic(Resource):
             end_datetime_timezone = timezone.localize(end_datetime)
             end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at < :end"
+            sql_query += " AND created_at < :end"
             arg_dict["end"] = end_datetime_utc
 
-        sql_query += " GROUP BY date order by date"
+        sql_query += " GROUP BY date ORDER BY date"
 
         response_data = []
 
@@ -133,14 +139,17 @@ class DailyTerminalsStatistic(Resource):
         account = current_user
 
         parser = reqparse.RequestParser()
-        parser.add_argument("start", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
-        parser.add_argument("end", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("start", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
         args = parser.parse_args()
 
-        sql_query = """
-                SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, count(distinct messages.from_end_user_id) AS terminal_count
-                    FROM messages where app_id = :app_id 
-                """
+        sql_query = """SELECT
+    DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date,
+    COUNT(DISTINCT messages.from_end_user_id) AS terminal_count
+FROM
+    messages
+WHERE
+    app_id = :app_id"""
         arg_dict = {"tz": account.timezone, "app_id": app_model.id}
 
         timezone = pytz.timezone(account.timezone)
@@ -153,7 +162,7 @@ class DailyTerminalsStatistic(Resource):
             start_datetime_timezone = timezone.localize(start_datetime)
             start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at >= :start"
+            sql_query += " AND created_at >= :start"
             arg_dict["start"] = start_datetime_utc
 
         if args["end"]:
@@ -163,10 +172,10 @@ class DailyTerminalsStatistic(Resource):
             end_datetime_timezone = timezone.localize(end_datetime)
             end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at < :end"
+            sql_query += " AND created_at < :end"
             arg_dict["end"] = end_datetime_utc
 
-        sql_query += " GROUP BY date order by date"
+        sql_query += " GROUP BY date ORDER BY date"
 
         response_data = []
 
@@ -187,16 +196,18 @@ class DailyTokenCostStatistic(Resource):
         account = current_user
 
         parser = reqparse.RequestParser()
-        parser.add_argument("start", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
-        parser.add_argument("end", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("start", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
         args = parser.parse_args()
 
-        sql_query = """
-                SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, 
-                    (sum(messages.message_tokens) + sum(messages.answer_tokens)) as token_count,
-                    sum(total_price) as total_price
-                    FROM messages where app_id = :app_id 
-                """
+        sql_query = """SELECT
+    DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date,
+    (SUM(messages.message_tokens) + SUM(messages.answer_tokens)) AS token_count,
+    SUM(total_price) AS total_price
+FROM
+    messages
+WHERE
+    app_id = :app_id"""
         arg_dict = {"tz": account.timezone, "app_id": app_model.id}
 
         timezone = pytz.timezone(account.timezone)
@@ -209,7 +220,7 @@ class DailyTokenCostStatistic(Resource):
             start_datetime_timezone = timezone.localize(start_datetime)
             start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at >= :start"
+            sql_query += " AND created_at >= :start"
             arg_dict["start"] = start_datetime_utc
 
         if args["end"]:
@@ -219,10 +230,10 @@ class DailyTokenCostStatistic(Resource):
             end_datetime_timezone = timezone.localize(end_datetime)
             end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at < :end"
+            sql_query += " AND created_at < :end"
             arg_dict["end"] = end_datetime_utc
 
-        sql_query += " GROUP BY date order by date"
+        sql_query += " GROUP BY date ORDER BY date"
 
         response_data = []
 
@@ -245,16 +256,26 @@ class AverageSessionInteractionStatistic(Resource):
         account = current_user
 
         parser = reqparse.RequestParser()
-        parser.add_argument("start", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
-        parser.add_argument("end", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("start", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
         args = parser.parse_args()
 
-        sql_query = """SELECT date(DATE_TRUNC('day', c.created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, 
-AVG(subquery.message_count) AS interactions
-FROM (SELECT m.conversation_id, COUNT(m.id) AS message_count
-    FROM conversations c
-    JOIN messages m ON c.id = m.conversation_id
-    WHERE c.override_model_configs IS NULL AND c.app_id = :app_id"""
+        sql_query = """SELECT
+    DATE(DATE_TRUNC('day', c.created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date,
+    AVG(subquery.message_count) AS interactions
+FROM
+    (
+        SELECT
+            m.conversation_id,
+            COUNT(m.id) AS message_count
+        FROM
+            conversations c
+        JOIN
+            messages m
+            ON c.id = m.conversation_id
+        WHERE
+            c.override_model_configs IS NULL
+            AND c.app_id = :app_id"""
         arg_dict = {"tz": account.timezone, "app_id": app_model.id}
 
         timezone = pytz.timezone(account.timezone)
@@ -267,7 +288,7 @@ FROM (SELECT m.conversation_id, COUNT(m.id) AS message_count
             start_datetime_timezone = timezone.localize(start_datetime)
             start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and c.created_at >= :start"
+            sql_query += " AND c.created_at >= :start"
             arg_dict["start"] = start_datetime_utc
 
         if args["end"]:
@@ -277,14 +298,19 @@ FROM (SELECT m.conversation_id, COUNT(m.id) AS message_count
             end_datetime_timezone = timezone.localize(end_datetime)
             end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and c.created_at < :end"
+            sql_query += " AND c.created_at < :end"
             arg_dict["end"] = end_datetime_utc
 
         sql_query += """
-        GROUP BY m.conversation_id) subquery
-LEFT JOIN conversations c on c.id=subquery.conversation_id
-GROUP BY date
-ORDER BY date"""
+        GROUP BY m.conversation_id
+    ) subquery
+LEFT JOIN
+    conversations c
+    ON c.id = subquery.conversation_id
+GROUP BY
+    date
+ORDER BY
+    date"""
 
         response_data = []
 
@@ -307,17 +333,21 @@ class UserSatisfactionRateStatistic(Resource):
         account = current_user
 
         parser = reqparse.RequestParser()
-        parser.add_argument("start", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
-        parser.add_argument("end", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("start", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
         args = parser.parse_args()
 
-        sql_query = """
-                        SELECT date(DATE_TRUNC('day', m.created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, 
-                            COUNT(m.id) as message_count, COUNT(mf.id) as feedback_count 
-                            FROM messages m
-                            LEFT JOIN message_feedbacks mf on mf.message_id=m.id and mf.rating='like'
-                            WHERE m.app_id = :app_id 
-                        """
+        sql_query = """SELECT
+    DATE(DATE_TRUNC('day', m.created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date,
+    COUNT(m.id) AS message_count,
+    COUNT(mf.id) AS feedback_count
+FROM
+    messages m
+LEFT JOIN
+    message_feedbacks mf
+    ON mf.message_id=m.id AND mf.rating='like'
+WHERE
+    m.app_id = :app_id"""
         arg_dict = {"tz": account.timezone, "app_id": app_model.id}
 
         timezone = pytz.timezone(account.timezone)
@@ -330,7 +360,7 @@ class UserSatisfactionRateStatistic(Resource):
             start_datetime_timezone = timezone.localize(start_datetime)
             start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and m.created_at >= :start"
+            sql_query += " AND m.created_at >= :start"
             arg_dict["start"] = start_datetime_utc
 
         if args["end"]:
@@ -340,10 +370,10 @@ class UserSatisfactionRateStatistic(Resource):
             end_datetime_timezone = timezone.localize(end_datetime)
             end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and m.created_at < :end"
+            sql_query += " AND m.created_at < :end"
             arg_dict["end"] = end_datetime_utc
 
-        sql_query += " GROUP BY date order by date"
+        sql_query += " GROUP BY date ORDER BY date"
 
         response_data = []
 
@@ -369,16 +399,17 @@ class AverageResponseTimeStatistic(Resource):
         account = current_user
 
         parser = reqparse.RequestParser()
-        parser.add_argument("start", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
-        parser.add_argument("end", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("start", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
         args = parser.parse_args()
 
-        sql_query = """
-                SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, 
-                    AVG(provider_response_latency) as latency
-                    FROM messages
-                    WHERE app_id = :app_id
-                """
+        sql_query = """SELECT
+    DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date,
+    AVG(provider_response_latency) AS latency
+FROM
+    messages
+WHERE
+    app_id = :app_id"""
         arg_dict = {"tz": account.timezone, "app_id": app_model.id}
 
         timezone = pytz.timezone(account.timezone)
@@ -391,7 +422,7 @@ class AverageResponseTimeStatistic(Resource):
             start_datetime_timezone = timezone.localize(start_datetime)
             start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at >= :start"
+            sql_query += " AND created_at >= :start"
             arg_dict["start"] = start_datetime_utc
 
         if args["end"]:
@@ -401,10 +432,10 @@ class AverageResponseTimeStatistic(Resource):
             end_datetime_timezone = timezone.localize(end_datetime)
             end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at < :end"
+            sql_query += " AND created_at < :end"
             arg_dict["end"] = end_datetime_utc
 
-        sql_query += " GROUP BY date order by date"
+        sql_query += " GROUP BY date ORDER BY date"
 
         response_data = []
 
@@ -425,17 +456,20 @@ class TokensPerSecondStatistic(Resource):
         account = current_user
 
         parser = reqparse.RequestParser()
-        parser.add_argument("start", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
-        parser.add_argument("end", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("start", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
         args = parser.parse_args()
 
-        sql_query = """SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, 
-    CASE 
+        sql_query = """SELECT
+    DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date,
+    CASE
         WHEN SUM(provider_response_latency) = 0 THEN 0
         ELSE (SUM(answer_tokens) / SUM(provider_response_latency))
     END as tokens_per_second
-FROM messages
-WHERE app_id = :app_id"""
+FROM
+    messages
+WHERE
+    app_id = :app_id"""
         arg_dict = {"tz": account.timezone, "app_id": app_model.id}
 
         timezone = pytz.timezone(account.timezone)
@@ -448,7 +482,7 @@ WHERE app_id = :app_id"""
             start_datetime_timezone = timezone.localize(start_datetime)
             start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at >= :start"
+            sql_query += " AND created_at >= :start"
             arg_dict["start"] = start_datetime_utc
 
         if args["end"]:
@@ -458,10 +492,10 @@ WHERE app_id = :app_id"""
             end_datetime_timezone = timezone.localize(end_datetime)
             end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at < :end"
+            sql_query += " AND created_at < :end"
             arg_dict["end"] = end_datetime_utc
 
-        sql_query += " GROUP BY date order by date"
+        sql_query += " GROUP BY date ORDER BY date"
 
         response_data = []
 

api/controllers/console/app/workflow.py

@@ -166,6 +166,8 @@ class AdvancedChatDraftWorkflowRunApi(Resource):
         parser.add_argument("query", type=str, required=True, location="json", default="")
         parser.add_argument("files", type=list, location="json")
         parser.add_argument("conversation_id", type=uuid_value, location="json")
+        parser.add_argument("parent_message_id", type=uuid_value, required=False, location="json")
+
         args = parser.parse_args()
 
         try:
@@ -465,6 +467,6 @@ api.add_resource(
 api.add_resource(PublishedWorkflowApi, "/apps/<uuid:app_id>/workflows/publish")
 api.add_resource(DefaultBlockConfigsApi, "/apps/<uuid:app_id>/workflows/default-workflow-block-configs")
 api.add_resource(
-    DefaultBlockConfigApi, "/apps/<uuid:app_id>/workflows/default-workflow-block-configs" "/<string:block_type>"
+    DefaultBlockConfigApi, "/apps/<uuid:app_id>/workflows/default-workflow-block-configs/<string:block_type>"
 )
 api.add_resource(ConvertToWorkflowApi, "/apps/<uuid:app_id>/convert-to-workflow")

api/controllers/console/app/workflow_statistic.py

@@ -11,7 +11,7 @@ from controllers.console.app.wraps import get_app_model
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
-from libs.helper import datetime_string
+from libs.helper import DatetimeString
 from libs.login import login_required
 from models.model import AppMode
 from models.workflow import WorkflowRunTriggeredFrom
@@ -26,16 +26,18 @@ class WorkflowDailyRunsStatistic(Resource):
         account = current_user
 
         parser = reqparse.RequestParser()
-        parser.add_argument("start", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
-        parser.add_argument("end", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("start", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
         args = parser.parse_args()
 
-        sql_query = """
-        SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, count(id) AS runs
-            FROM workflow_runs 
-            WHERE app_id = :app_id 
-                AND triggered_from = :triggered_from
-        """
+        sql_query = """SELECT
+    DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date,
+    COUNT(id) AS runs
+FROM
+    workflow_runs
+WHERE
+    app_id = :app_id
+    AND triggered_from = :triggered_from"""
         arg_dict = {
             "tz": account.timezone,
             "app_id": app_model.id,
@@ -52,7 +54,7 @@ class WorkflowDailyRunsStatistic(Resource):
             start_datetime_timezone = timezone.localize(start_datetime)
             start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at >= :start"
+            sql_query += " AND created_at >= :start"
             arg_dict["start"] = start_datetime_utc
 
         if args["end"]:
@@ -62,10 +64,10 @@ class WorkflowDailyRunsStatistic(Resource):
             end_datetime_timezone = timezone.localize(end_datetime)
             end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at < :end"
+            sql_query += " AND created_at < :end"
             arg_dict["end"] = end_datetime_utc
 
-        sql_query += " GROUP BY date order by date"
+        sql_query += " GROUP BY date ORDER BY date"
 
         response_data = []
 
@@ -86,16 +88,18 @@ class WorkflowDailyTerminalsStatistic(Resource):
         account = current_user
 
         parser = reqparse.RequestParser()
-        parser.add_argument("start", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
-        parser.add_argument("end", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("start", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
         args = parser.parse_args()
 
-        sql_query = """
-                SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, count(distinct workflow_runs.created_by) AS terminal_count
-                    FROM workflow_runs 
-                    WHERE app_id = :app_id 
-                        AND triggered_from = :triggered_from
-                """
+        sql_query = """SELECT
+    DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date,
+    COUNT(DISTINCT workflow_runs.created_by) AS terminal_count
+FROM
+    workflow_runs
+WHERE
+    app_id = :app_id
+    AND triggered_from = :triggered_from"""
         arg_dict = {
             "tz": account.timezone,
             "app_id": app_model.id,
@@ -112,7 +116,7 @@ class WorkflowDailyTerminalsStatistic(Resource):
             start_datetime_timezone = timezone.localize(start_datetime)
             start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at >= :start"
+            sql_query += " AND created_at >= :start"
             arg_dict["start"] = start_datetime_utc
 
         if args["end"]:
@@ -122,10 +126,10 @@ class WorkflowDailyTerminalsStatistic(Resource):
             end_datetime_timezone = timezone.localize(end_datetime)
             end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at < :end"
+            sql_query += " AND created_at < :end"
             arg_dict["end"] = end_datetime_utc
 
-        sql_query += " GROUP BY date order by date"
+        sql_query += " GROUP BY date ORDER BY date"
 
         response_data = []
 
@@ -146,18 +150,18 @@ class WorkflowDailyTokenCostStatistic(Resource):
         account = current_user
 
         parser = reqparse.RequestParser()
-        parser.add_argument("start", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
-        parser.add_argument("end", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("start", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
         args = parser.parse_args()
 
-        sql_query = """
-                SELECT 
-                    date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, 
-                    SUM(workflow_runs.total_tokens) as token_count
-                FROM workflow_runs 
-                WHERE app_id = :app_id 
-                    AND triggered_from = :triggered_from
-                """
+        sql_query = """SELECT
+    DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date,
+    SUM(workflow_runs.total_tokens) AS token_count
+FROM
+    workflow_runs
+WHERE
+    app_id = :app_id
+    AND triggered_from = :triggered_from"""
         arg_dict = {
             "tz": account.timezone,
             "app_id": app_model.id,
@@ -174,7 +178,7 @@ class WorkflowDailyTokenCostStatistic(Resource):
             start_datetime_timezone = timezone.localize(start_datetime)
             start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at >= :start"
+            sql_query += " AND created_at >= :start"
             arg_dict["start"] = start_datetime_utc
 
         if args["end"]:
@@ -184,10 +188,10 @@ class WorkflowDailyTokenCostStatistic(Resource):
             end_datetime_timezone = timezone.localize(end_datetime)
             end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query += " and created_at < :end"
+            sql_query += " AND created_at < :end"
             arg_dict["end"] = end_datetime_utc
 
-        sql_query += " GROUP BY date order by date"
+        sql_query += " GROUP BY date ORDER BY date"
 
         response_data = []
 
@@ -213,27 +217,31 @@ class WorkflowAverageAppInteractionStatistic(Resource):
         account = current_user
 
         parser = reqparse.RequestParser()
-        parser.add_argument("start", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
-        parser.add_argument("end", type=datetime_string("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("start", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
+        parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
         args = parser.parse_args()
 
-        sql_query = """
-            SELECT 
-                AVG(sub.interactions) as interactions,
-                sub.date
-            FROM
-                (SELECT 
-                    date(DATE_TRUNC('day', c.created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, 
-                    c.created_by,
-                    COUNT(c.id) AS interactions
-                FROM workflow_runs c
-                WHERE c.app_id = :app_id
-                    AND c.triggered_from = :triggered_from
-                    {{start}}
-                    {{end}}
-                GROUP BY date, c.created_by) sub
-            GROUP BY sub.date
-            """
+        sql_query = """SELECT
+    AVG(sub.interactions) AS interactions,
+    sub.date
+FROM
+    (
+        SELECT
+            DATE(DATE_TRUNC('day', c.created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date,
+            c.created_by,
+            COUNT(c.id) AS interactions
+        FROM
+            workflow_runs c
+        WHERE
+            c.app_id = :app_id
+            AND c.triggered_from = :triggered_from
+            {{start}}
+            {{end}}
+        GROUP BY
+            date, c.created_by
+    ) sub
+GROUP BY
+    sub.date"""
         arg_dict = {
             "tz": account.timezone,
             "app_id": app_model.id,
@@ -262,7 +270,7 @@ class WorkflowAverageAppInteractionStatistic(Resource):
             end_datetime_timezone = timezone.localize(end_datetime)
             end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
 
-            sql_query = sql_query.replace("{{end}}", " and c.created_at < :end")
+            sql_query = sql_query.replace("{{end}}", " AND c.created_at < :end")
             arg_dict["end"] = end_datetime_utc
         else:
             sql_query = sql_query.replace("{{end}}", "")

api/controllers/console/auth/activate.py

@@ -8,7 +8,7 @@ from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.error import AlreadyActivateError
 from extensions.ext_database import db
-from libs.helper import email, str_len, timezone
+from libs.helper import StrLen, email, timezone
 from libs.password import hash_password, valid_password
 from models.account import AccountStatus
 from services.account_service import RegisterService
@@ -37,7 +37,7 @@ class ActivateApi(Resource):
         parser.add_argument("workspace_id", type=str, required=False, nullable=True, location="json")
         parser.add_argument("email", type=email, required=False, nullable=True, location="json")
         parser.add_argument("token", type=str, required=True, nullable=False, location="json")
-        parser.add_argument("name", type=str_len(30), required=True, nullable=False, location="json")
+        parser.add_argument("name", type=StrLen(30), required=True, nullable=False, location="json")
         parser.add_argument("password", type=valid_password, required=True, nullable=False, location="json")
         parser.add_argument(
             "interface_language", type=supported_language, required=True, nullable=False, location="json"

api/controllers/console/auth/oauth.py

@@ -71,7 +71,7 @@ class OAuthCallback(Resource):
 
         account = _generate_account(provider, user_info)
         # Check account status
-        if account.status == AccountStatus.BANNED.value or account.status == AccountStatus.CLOSED.value:
+        if account.status in {AccountStatus.BANNED.value, AccountStatus.CLOSED.value}:
             return {"error": "Account is banned or closed."}, 403
 
         if account.status == AccountStatus.PENDING.value:
@@ -101,7 +101,7 @@ def _generate_account(provider: str, user_info: OAuthUserInfo):
 
     if not account:
         # Create account
-        account_name = user_info.name if user_info.name else "Dify"
+        account_name = user_info.name or "Dify"
         account = RegisterService.register(
             email=user_info.email, name=account_name, password=None, open_id=user_info.id, provider=provider
         )

api/controllers/console/datasets/datasets.py

@@ -18,7 +18,7 @@ from core.model_runtime.entities.model_entities import ModelType
 from core.provider_manager import ProviderManager
 from core.rag.datasource.vdb.vector_type import VectorType
 from core.rag.extractor.entity.extract_setting import ExtractSetting
-from core.rag.retrieval.retrival_methods import RetrievalMethod
+from core.rag.retrieval.retrieval_methods import RetrievalMethod
 from extensions.ext_database import db
 from fields.app_fields import related_app_list
 from fields.dataset_fields import dataset_detail_fields, dataset_query_detail_fields
@@ -49,7 +49,7 @@ class DatasetListApi(Resource):
         page = request.args.get("page", default=1, type=int)
         limit = request.args.get("limit", default=20, type=int)
         ids = request.args.getlist("ids")
-        provider = request.args.get("provider", default="vendor")
+        # provider = request.args.get("provider", default="vendor")
         search = request.args.get("keyword", default=None, type=str)
         tag_ids = request.args.getlist("tag_ids")
 
@@ -57,7 +57,7 @@ class DatasetListApi(Resource):
             datasets, total = DatasetService.get_datasets_by_ids(ids, current_user.current_tenant_id)
         else:
             datasets, total = DatasetService.get_datasets(
-                page, limit, provider, current_user.current_tenant_id, current_user, search, tag_ids
+                page, limit, current_user.current_tenant_id, current_user, search, tag_ids
             )
 
         # check embedding setting
@@ -110,6 +110,26 @@ class DatasetListApi(Resource):
             nullable=True,
             help="Invalid indexing technique.",
         )
+        parser.add_argument(
+            "external_knowledge_api_id",
+            type=str,
+            nullable=True,
+            required=False,
+        )
+        parser.add_argument(
+            "provider",
+            type=str,
+            nullable=True,
+            choices=Dataset.PROVIDER_LIST,
+            required=False,
+            default="vendor",
+        )
+        parser.add_argument(
+            "external_knowledge_id",
+            type=str,
+            nullable=True,
+            required=False,
+        )
         args = parser.parse_args()
 
         # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
@@ -123,6 +143,9 @@ class DatasetListApi(Resource):
                 indexing_technique=args["indexing_technique"],
                 account=current_user,
                 permission=DatasetPermissionEnum.ONLY_ME,
+                provider=args["provider"],
+                external_knowledge_api_id=args["external_knowledge_api_id"],
+                external_knowledge_id=args["external_knowledge_id"],
             )
         except services.errors.dataset.DatasetNameDuplicateError:
             raise DatasetNameDuplicateError()
@@ -211,6 +234,33 @@ class DatasetApi(Resource):
         )
         parser.add_argument("retrieval_model", type=dict, location="json", help="Invalid retrieval model.")
         parser.add_argument("partial_member_list", type=list, location="json", help="Invalid parent user list.")
+
+        parser.add_argument(
+            "external_retrieval_model",
+            type=dict,
+            required=False,
+            nullable=True,
+            location="json",
+            help="Invalid external retrieval model.",
+        )
+
+        parser.add_argument(
+            "external_knowledge_id",
+            type=str,
+            required=False,
+            nullable=True,
+            location="json",
+            help="Invalid external knowledge id.",
+        )
+
+        parser.add_argument(
+            "external_knowledge_api_id",
+            type=str,
+            required=False,
+            nullable=True,
+            location="json",
+            help="Invalid external knowledge api id.",
+        )
         args = parser.parse_args()
         data = request.get_json()
 
@@ -399,7 +449,7 @@ class DatasetIndexingEstimateApi(Resource):
             )
         except LLMBadRequestError:
             raise ProviderNotInitializeError(
-                "No Embedding Model available. Please configure a valid provider " "in the Settings -> Model Provider."
+                "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
@@ -550,12 +600,7 @@ class DatasetApiBaseUrlApi(Resource):
     @login_required
     @account_initialization_required
     def get(self):
-        return {
-            "api_base_url": (
-                dify_config.SERVICE_API_URL if dify_config.SERVICE_API_URL else request.host_url.rstrip("/")
-            )
-            + "/v1"
-        }
+        return {"api_base_url": (dify_config.SERVICE_API_URL or request.host_url.rstrip("/")) + "/v1"}
 
 
 class DatasetRetrievalSettingApi(Resource):
@@ -568,10 +613,10 @@ class DatasetRetrievalSettingApi(Resource):
             case (
                 VectorType.MILVUS
                 | VectorType.RELYT
-                | VectorType.PGVECTOR
                 | VectorType.TIDB_VECTOR
                 | VectorType.CHROMA
                 | VectorType.TENCENT
+                | VectorType.PGVECTO_RS
             ):
                 return {"retrieval_method": [RetrievalMethod.SEMANTIC_SEARCH.value]}
             case (
@@ -582,6 +627,7 @@ class DatasetRetrievalSettingApi(Resource):
                 | VectorType.MYSCALE
                 | VectorType.ORACLE
                 | VectorType.ELASTICSEARCH
+                | VectorType.PGVECTOR
             ):
                 return {
                     "retrieval_method": [

api/controllers/console/datasets/datasets_document.py

@@ -302,6 +302,8 @@ class DatasetInitApi(Resource):
             "doc_language", type=str, default="English", required=False, nullable=False, location="json"
         )
         parser.add_argument("retrieval_model", type=dict, required=False, nullable=False, location="json")
+        parser.add_argument("embedding_model", type=str, required=False, nullable=True, location="json")
+        parser.add_argument("embedding_model_provider", type=str, required=False, nullable=True, location="json")
         args = parser.parse_args()
 
         # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
@@ -309,6 +311,8 @@ class DatasetInitApi(Resource):
             raise Forbidden()
 
         if args["indexing_technique"] == "high_quality":
+            if args["embedding_model"] is None or args["embedding_model_provider"] is None:
+                raise ValueError("embedding model and embedding model provider are required for high quality indexing.")
             try:
                 model_manager = ModelManager()
                 model_manager.get_default_model_instance(
@@ -350,7 +354,7 @@ class DocumentIndexingEstimateApi(DocumentResource):
         document_id = str(document_id)
         document = self.get_document(dataset_id, document_id)
 
-        if document.indexing_status in ["completed", "error"]:
+        if document.indexing_status in {"completed", "error"}:
             raise DocumentAlreadyFinishedError()
 
         data_process_rule = document.dataset_process_rule
@@ -417,7 +421,7 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
         info_list = []
         extract_settings = []
         for document in documents:
-            if document.indexing_status in ["completed", "error"]:
+            if document.indexing_status in {"completed", "error"}:
                 raise DocumentAlreadyFinishedError()
             data_source_info = document.data_source_info_dict
             # format document files info
@@ -661,7 +665,7 @@ class DocumentProcessingApi(DocumentResource):
             db.session.commit()
 
         elif action == "resume":
-            if document.indexing_status not in ["paused", "error"]:
+            if document.indexing_status not in {"paused", "error"}:
                 raise InvalidActionError("Document not in paused or error state.")
 
             document.paused_by = None

api/controllers/console/datasets/external.py

@@ -0,0 +1,239 @@
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, marshal, reqparse
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
+
+import services
+from controllers.console import api
+from controllers.console.datasets.error import DatasetNameDuplicateError
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from fields.dataset_fields import dataset_detail_fields
+from libs.login import login_required
+from services.dataset_service import DatasetService
+from services.external_knowledge_service import ExternalDatasetService
+from services.hit_testing_service import HitTestingService
+
+
+def _validate_name(name):
+    if not name or len(name) < 1 or len(name) > 100:
+        raise ValueError("Name must be between 1 to 100 characters.")
+    return name
+
+
+def _validate_description_length(description):
+    if description and len(description) > 400:
+        raise ValueError("Description cannot exceed 400 characters.")
+    return description
+
+
+class ExternalApiTemplateListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        page = request.args.get("page", default=1, type=int)
+        limit = request.args.get("limit", default=20, type=int)
+        search = request.args.get("keyword", default=None, type=str)
+
+        external_knowledge_apis, total = ExternalDatasetService.get_external_knowledge_apis(
+            page, limit, current_user.current_tenant_id, search
+        )
+        response = {
+            "data": [item.to_dict() for item in external_knowledge_apis],
+            "has_more": len(external_knowledge_apis) == limit,
+            "limit": limit,
+            "total": total,
+            "page": page,
+        }
+        return response, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument(
+            "name",
+            nullable=False,
+            required=True,
+            help="Name is required. Name must be between 1 to 100 characters.",
+            type=_validate_name,
+        )
+        parser.add_argument(
+            "settings",
+            type=dict,
+            location="json",
+            nullable=False,
+            required=True,
+        )
+        args = parser.parse_args()
+
+        ExternalDatasetService.validate_api_list(args["settings"])
+
+        # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+
+        try:
+            external_knowledge_api = ExternalDatasetService.create_external_knowledge_api(
+                tenant_id=current_user.current_tenant_id, user_id=current_user.id, args=args
+            )
+        except services.errors.dataset.DatasetNameDuplicateError:
+            raise DatasetNameDuplicateError()
+
+        return external_knowledge_api.to_dict(), 201
+
+
+class ExternalApiTemplateApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, external_knowledge_api_id):
+        external_knowledge_api_id = str(external_knowledge_api_id)
+        external_knowledge_api = ExternalDatasetService.get_external_knowledge_api(external_knowledge_api_id)
+        if external_knowledge_api is None:
+            raise NotFound("API template not found.")
+
+        return external_knowledge_api.to_dict(), 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def patch(self, external_knowledge_api_id):
+        external_knowledge_api_id = str(external_knowledge_api_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument(
+            "name",
+            nullable=False,
+            required=True,
+            help="type is required. Name must be between 1 to 100 characters.",
+            type=_validate_name,
+        )
+        parser.add_argument(
+            "settings",
+            type=dict,
+            location="json",
+            nullable=False,
+            required=True,
+        )
+        args = parser.parse_args()
+        ExternalDatasetService.validate_api_list(args["settings"])
+
+        external_knowledge_api = ExternalDatasetService.update_external_knowledge_api(
+            tenant_id=current_user.current_tenant_id,
+            user_id=current_user.id,
+            external_knowledge_api_id=external_knowledge_api_id,
+            args=args,
+        )
+
+        return external_knowledge_api.to_dict(), 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, external_knowledge_api_id):
+        external_knowledge_api_id = str(external_knowledge_api_id)
+
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor or current_user.is_dataset_operator:
+            raise Forbidden()
+
+        ExternalDatasetService.delete_external_knowledge_api(current_user.current_tenant_id, external_knowledge_api_id)
+        return {"result": "success"}, 200
+
+
+class ExternalApiUseCheckApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, external_knowledge_api_id):
+        external_knowledge_api_id = str(external_knowledge_api_id)
+
+        external_knowledge_api_is_using, count = ExternalDatasetService.external_knowledge_api_use_check(
+            external_knowledge_api_id
+        )
+        return {"is_using": external_knowledge_api_is_using, "count": count}, 200
+
+
+class ExternalDatasetCreateApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("external_knowledge_api_id", type=str, required=True, nullable=False, location="json")
+        parser.add_argument("external_knowledge_id", type=str, required=True, nullable=False, location="json")
+        parser.add_argument(
+            "name",
+            nullable=False,
+            required=True,
+            help="name is required. Name must be between 1 to 100 characters.",
+            type=_validate_name,
+        )
+        parser.add_argument("description", type=str, required=False, nullable=True, location="json")
+        parser.add_argument("external_retrieval_model", type=dict, required=False, location="json")
+
+        args = parser.parse_args()
+
+        # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+
+        try:
+            dataset = ExternalDatasetService.create_external_dataset(
+                tenant_id=current_user.current_tenant_id,
+                user_id=current_user.id,
+                args=args,
+            )
+        except services.errors.dataset.DatasetNameDuplicateError:
+            raise DatasetNameDuplicateError()
+
+        return marshal(dataset, dataset_detail_fields), 201
+
+
+class ExternalKnowledgeHitTestingApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("query", type=str, location="json")
+        parser.add_argument("external_retrieval_model", type=dict, required=False, location="json")
+        args = parser.parse_args()
+
+        HitTestingService.hit_testing_args_check(args)
+
+        try:
+            response = HitTestingService.external_retrieve(
+                dataset=dataset,
+                query=args["query"],
+                account=current_user,
+                external_retrieval_model=args["external_retrieval_model"],
+            )
+
+            return response
+        except Exception as e:
+            raise InternalServerError(str(e))
+
+
+api.add_resource(ExternalKnowledgeHitTestingApi, "/datasets/<uuid:dataset_id>/external-hit-testing")
+api.add_resource(ExternalDatasetCreateApi, "/datasets/external")
+api.add_resource(ExternalApiTemplateListApi, "/datasets/external-knowledge-api")
+api.add_resource(ExternalApiTemplateApi, "/datasets/external-knowledge-api/<uuid:external_knowledge_api_id>")
+api.add_resource(ExternalApiUseCheckApi, "/datasets/external-knowledge-api/<uuid:external_knowledge_api_id>/use-check")

api/controllers/console/datasets/hit_testing.py

@@ -47,6 +47,7 @@ class HitTestingApi(Resource):
         parser = reqparse.RequestParser()
         parser.add_argument("query", type=str, location="json")
         parser.add_argument("retrieval_model", type=dict, required=False, location="json")
+        parser.add_argument("external_retrieval_model", type=dict, required=False, location="json")
         args = parser.parse_args()
 
         HitTestingService.hit_testing_args_check(args)
@@ -57,6 +58,7 @@ class HitTestingApi(Resource):
                 query=args["query"],
                 account=current_user,
                 retrieval_model=args["retrieval_model"],
+                external_retrieval_model=args["external_retrieval_model"],
                 limit=10,
             )
 

api/controllers/console/datasets/website.py

@@ -14,7 +14,9 @@ class WebsiteCrawlApi(Resource):
     @account_initialization_required
     def post(self):
         parser = reqparse.RequestParser()
-        parser.add_argument("provider", type=str, choices=["firecrawl"], required=True, nullable=True, location="json")
+        parser.add_argument(
+            "provider", type=str, choices=["firecrawl", "jinareader"], required=True, nullable=True, location="json"
+        )
         parser.add_argument("url", type=str, required=True, nullable=True, location="json")
         parser.add_argument("options", type=dict, required=True, nullable=True, location="json")
         args = parser.parse_args()
@@ -33,7 +35,7 @@ class WebsiteCrawlStatusApi(Resource):
     @account_initialization_required
     def get(self, job_id: str):
         parser = reqparse.RequestParser()
-        parser.add_argument("provider", type=str, choices=["firecrawl"], required=True, location="args")
+        parser.add_argument("provider", type=str, choices=["firecrawl", "jinareader"], required=True, location="args")
         args = parser.parse_args()
         # get crawl status
         try:

api/controllers/console/error.py

@@ -18,9 +18,7 @@ class NotSetupError(BaseHTTPException):
 
 class NotInitValidateError(BaseHTTPException):
     error_code = "not_init_validated"
-    description = (
-        "Init validation has not been completed yet. " "Please proceed with the init validation process first."
-    )
+    description = "Init validation has not been completed yet. Please proceed with the init validation process first."
     code = 401
 
 

api/controllers/console/explore/audio.py

@@ -81,19 +81,15 @@ class ChatTextApi(InstalledAppResource):
             message_id = args.get("message_id", None)
             text = args.get("text", None)
             if (
-                app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]
+                app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}
                 and app_model.workflow
                 and app_model.workflow.features_dict
             ):
                 text_to_speech = app_model.workflow.features_dict.get("text_to_speech")
-                voice = args.get("voice") if args.get("voice") else text_to_speech.get("voice")
+                voice = args.get("voice") or text_to_speech.get("voice")
             else:
                 try:
-                    voice = (
-                        args.get("voice")
-                        if args.get("voice")
-                        else app_model.app_model_config.text_to_speech_dict.get("voice")
-                    )
+                    voice = args.get("voice") or app_model.app_model_config.text_to_speech_dict.get("voice")
                 except Exception:
                     voice = None
             response = AudioService.transcript_tts(app_model=app_model, message_id=message_id, voice=voice, text=text)

api/controllers/console/explore/completion.py

@@ -92,7 +92,7 @@ class ChatApi(InstalledAppResource):
     def post(self, installed_app):
         app_model = installed_app.app
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -100,6 +100,7 @@ class ChatApi(InstalledAppResource):
         parser.add_argument("query", type=str, required=True, location="json")
         parser.add_argument("files", type=list, required=False, location="json")
         parser.add_argument("conversation_id", type=uuid_value, location="json")
+        parser.add_argument("parent_message_id", type=uuid_value, required=False, location="json")
         parser.add_argument("retriever_from", type=str, required=False, default="explore_app", location="json")
         args = parser.parse_args()
 
@@ -140,7 +141,7 @@ class ChatStopApi(InstalledAppResource):
     def post(self, installed_app, task_id):
         app_model = installed_app.app
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         AppQueueManager.set_stop_flag(task_id, InvokeFrom.EXPLORE, current_user.id)

api/controllers/console/explore/conversation.py

@@ -20,7 +20,7 @@ class ConversationListApi(InstalledAppResource):
     def get(self, installed_app):
         app_model = installed_app.app
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -50,7 +50,7 @@ class ConversationApi(InstalledAppResource):
     def delete(self, installed_app, c_id):
         app_model = installed_app.app
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -68,7 +68,7 @@ class ConversationRenameApi(InstalledAppResource):
     def post(self, installed_app, c_id):
         app_model = installed_app.app
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -90,7 +90,7 @@ class ConversationPinApi(InstalledAppResource):
     def patch(self, installed_app, c_id):
         app_model = installed_app.app
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -107,7 +107,7 @@ class ConversationUnPinApi(InstalledAppResource):
     def patch(self, installed_app, c_id):
         app_model = installed_app.app
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)

api/controllers/console/explore/installed_app.py

@@ -31,7 +31,7 @@ class InstalledAppsListApi(Resource):
                 "app_owner_tenant_id": installed_app.app_owner_tenant_id,
                 "is_pinned": installed_app.is_pinned,
                 "last_used_at": installed_app.last_used_at,
-                "editable": current_user.role in ["owner", "admin"],
+                "editable": current_user.role in {"owner", "admin"},
                 "uninstallable": current_tenant_id == installed_app.app_owner_tenant_id,
             }
             for installed_app in installed_apps

api/controllers/console/explore/message.py

@@ -40,7 +40,7 @@ class MessageListApi(InstalledAppResource):
         app_model = installed_app.app
 
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -51,7 +51,7 @@ class MessageListApi(InstalledAppResource):
 
         try:
             return MessageService.pagination_by_first_id(
-                app_model, current_user, args["conversation_id"], args["first_id"], args["limit"]
+                app_model, current_user, args["conversation_id"], args["first_id"], args["limit"], "desc"
             )
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
@@ -125,7 +125,7 @@ class MessageSuggestedQuestionApi(InstalledAppResource):
     def get(self, installed_app, message_id):
         app_model = installed_app.app
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         message_id = str(message_id)

api/controllers/console/explore/parameter.py

@@ -43,7 +43,7 @@ class AppParameterApi(InstalledAppResource):
         """Retrieve app parameters."""
         app_model = installed_app.app
 
-        if app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]:
+        if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}:
             workflow = app_model.workflow
             if workflow is None:
                 raise AppUnavailableError()

api/controllers/console/init_validate.py

@@ -4,7 +4,7 @@ from flask import session
 from flask_restful import Resource, reqparse
 
 from configs import dify_config
-from libs.helper import str_len
+from libs.helper import StrLen
 from models.model import DifySetup
 from services.account_service import TenantService
 
@@ -28,7 +28,7 @@ class InitValidateAPI(Resource):
             raise AlreadySetupError()
 
         parser = reqparse.RequestParser()
-        parser.add_argument("password", type=str_len(30), required=True, location="json")
+        parser.add_argument("password", type=StrLen(30), required=True, location="json")
         input_password = parser.parse_args()["password"]
 
         if input_password != os.environ.get("INIT_PASSWORD"):

api/controllers/console/setup.py

@@ -4,7 +4,7 @@ from flask import request
 from flask_restful import Resource, reqparse
 
 from configs import dify_config
-from libs.helper import email, get_remote_ip, str_len
+from libs.helper import StrLen, email, get_remote_ip
 from libs.password import valid_password
 from models.model import DifySetup
 from services.account_service import RegisterService, TenantService
@@ -40,7 +40,7 @@ class SetupApi(Resource):
 
         parser = reqparse.RequestParser()
         parser.add_argument("email", type=email, required=True, location="json")
-        parser.add_argument("name", type=str_len(30), required=True, location="json")
+        parser.add_argument("name", type=StrLen(30), required=True, location="json")
         parser.add_argument("password", type=valid_password, required=True, location="json")
         args = parser.parse_args()
 

api/controllers/console/tag/tags.py

@@ -13,7 +13,7 @@ from services.tag_service import TagService
 
 
 def _validate_name(name):
-    if not name or len(name) < 1 or len(name) > 40:
+    if not name or len(name) < 1 or len(name) > 50:
         raise ValueError("Name must be between 1 to 50 characters.")
     return name
 

api/controllers/console/version.py

@@ -38,11 +38,52 @@ class VersionApi(Resource):
             return result
 
         content = json.loads(response.content)
-        result["version"] = content["version"]
-        result["release_date"] = content["releaseDate"]
-        result["release_notes"] = content["releaseNotes"]
-        result["can_auto_update"] = content["canAutoUpdate"]
+        if _has_new_version(latest_version=content["version"], current_version=f"{args.get('current_version')}"):
+            result["version"] = content["version"]
+            result["release_date"] = content["releaseDate"]
+            result["release_notes"] = content["releaseNotes"]
+            result["can_auto_update"] = content["canAutoUpdate"]
         return result
 
 
+def _has_new_version(*, latest_version: str, current_version: str) -> bool:
+    def parse_version(version: str) -> tuple:
+        # Split version into parts and pre-release suffix if any
+        parts = version.split("-")
+        version_parts = parts[0].split(".")
+        pre_release = parts[1] if len(parts) > 1 else None
+
+        # Validate version format
+        if len(version_parts) != 3:
+            raise ValueError(f"Invalid version format: {version}")
+
+        try:
+            # Convert version parts to integers
+            major, minor, patch = map(int, version_parts)
+            return (major, minor, patch, pre_release)
+        except ValueError:
+            raise ValueError(f"Invalid version format: {version}")
+
+    latest = parse_version(latest_version)
+    current = parse_version(current_version)
+
+    # Compare major, minor, and patch versions
+    for latest_part, current_part in zip(latest[:3], current[:3]):
+        if latest_part > current_part:
+            return True
+        elif latest_part < current_part:
+            return False
+
+    # If versions are equal, check pre-release suffixes
+    if latest[3] is None and current[3] is not None:
+        return True
+    elif latest[3] is not None and current[3] is None:
+        return False
+    elif latest[3] is not None and current[3] is not None:
+        # Simple string comparison for pre-release versions
+        return latest[3] > current[3]
+
+    return False
+
+
 api.add_resource(VersionApi, "/version")

api/controllers/console/workspace/model_providers.py

@@ -218,7 +218,7 @@ api.add_resource(ModelProviderCredentialApi, "/workspaces/current/model-provider
 api.add_resource(ModelProviderValidateApi, "/workspaces/current/model-providers/<string:provider>/credentials/validate")
 api.add_resource(ModelProviderApi, "/workspaces/current/model-providers/<string:provider>")
 api.add_resource(
-    ModelProviderIconApi, "/workspaces/current/model-providers/<string:provider>/" "<string:icon_type>/<string:lang>"
+    ModelProviderIconApi, "/workspaces/current/model-providers/<string:provider>/<string:icon_type>/<string:lang>"
 )
 
 api.add_resource(

api/controllers/console/workspace/models.py

@@ -72,8 +72,9 @@ class DefaultModelApi(Resource):
                     provider=model_setting["provider"],
                     model=model_setting["model"],
                 )
-            except Exception:
-                logging.warning(f"{model_setting['model_type']} save error")
+            except Exception as ex:
+                logging.exception(f"{model_setting['model_type']} save error: {ex}")
+                raise ex
 
         return {"result": "success"}
 

api/controllers/console/workspace/tool_providers.py

@@ -327,7 +327,7 @@ class ToolApiProviderPreviousTestApi(Resource):
 
         return ApiToolManageService.test_api_tool_preview(
             current_user.current_tenant_id,
-            args["provider_name"] if args["provider_name"] else "",
+            args["provider_name"] or "",
             args["tool_name"],
             args["credentials"],
             args["parameters"],

api/controllers/console/workspace/workspace.py

@@ -194,7 +194,7 @@ class WebappLogoWorkspaceApi(Resource):
             raise TooManyFilesError()
 
         extension = file.filename.split(".")[-1]
-        if extension.lower() not in ["svg", "png"]:
+        if extension.lower() not in {"svg", "png"}:
             raise UnsupportedFileTypeError()
 
         try:

api/controllers/console/wraps.py

@@ -64,7 +64,8 @@ def cloud_edition_billing_resource_check(resource: str):
                 elif resource == "vector_space" and 0 < vector_space.limit <= vector_space.size:
                     abort(403, "The capacity of the vector space has reached the limit of your subscription.")
                 elif resource == "documents" and 0 < documents_upload_quota.limit <= documents_upload_quota.size:
-                    # The api of file upload is used in the multiple places, so we need to check the source of the request from datasets
+                    # The api of file upload is used in the multiple places,
+                    # so we need to check the source of the request from datasets
                     source = request.args.get("source")
                     if source == "datasets":
                         abort(403, "The number of documents has reached the limit of your subscription.")

api/controllers/service_api/app/app.py

@@ -42,7 +42,7 @@ class AppParameterApi(Resource):
     @marshal_with(parameters_fields)
     def get(self, app_model: App):
         """Retrieve app parameters."""
-        if app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]:
+        if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}:
             workflow = app_model.workflow
             if workflow is None:
                 raise AppUnavailableError()

api/controllers/service_api/app/audio.py

@@ -79,19 +79,15 @@ class TextApi(Resource):
             message_id = args.get("message_id", None)
             text = args.get("text", None)
             if (
-                app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]
+                app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}
                 and app_model.workflow
                 and app_model.workflow.features_dict
             ):
                 text_to_speech = app_model.workflow.features_dict.get("text_to_speech")
-                voice = args.get("voice") if args.get("voice") else text_to_speech.get("voice")
+                voice = args.get("voice") or text_to_speech.get("voice")
             else:
                 try:
-                    voice = (
-                        args.get("voice")
-                        if args.get("voice")
-                        else app_model.app_model_config.text_to_speech_dict.get("voice")
-                    )
+                    voice = args.get("voice") or app_model.app_model_config.text_to_speech_dict.get("voice")
                 except Exception:
                     voice = None
             response = AudioService.transcript_tts(

api/controllers/service_api/app/completion.py

@@ -96,7 +96,7 @@ class ChatApi(Resource):
     @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON, required=True))
     def post(self, app_model: App, end_user: EndUser):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -144,7 +144,7 @@ class ChatStopApi(Resource):
     @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON, required=True))
     def post(self, app_model: App, end_user: EndUser, task_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         AppQueueManager.set_stop_flag(task_id, InvokeFrom.SERVICE_API, end_user.id)

api/controllers/service_api/app/conversation.py

@@ -18,7 +18,7 @@ class ConversationApi(Resource):
     @marshal_with(conversation_infinite_scroll_pagination_fields)
     def get(self, app_model: App, end_user: EndUser):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -52,7 +52,7 @@ class ConversationDetailApi(Resource):
     @marshal_with(simple_conversation_fields)
     def delete(self, app_model: App, end_user: EndUser, c_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -69,7 +69,7 @@ class ConversationRenameApi(Resource):
     @marshal_with(simple_conversation_fields)
     def post(self, app_model: App, end_user: EndUser, c_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)

api/controllers/service_api/app/message.py

@@ -54,6 +54,7 @@ class MessageListApi(Resource):
     message_fields = {
         "id": fields.String,
         "conversation_id": fields.String,
+        "parent_message_id": fields.String,
         "inputs": fields.Raw,
         "query": fields.String,
         "answer": fields.String(attribute="re_sign_file_url_answer"),
@@ -76,7 +77,7 @@ class MessageListApi(Resource):
     @marshal_with(message_infinite_scroll_pagination_fields)
     def get(self, app_model: App, end_user: EndUser):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -117,7 +118,7 @@ class MessageSuggestedApi(Resource):
     def get(self, app_model: App, end_user: EndUser, message_id):
         message_id = str(message_id)
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         try:

api/controllers/service_api/app/workflow.py

@@ -1,6 +1,7 @@
 import logging
 
 from flask_restful import Resource, fields, marshal_with, reqparse
+from flask_restful.inputs import int_range
 from werkzeug.exceptions import InternalServerError
 
 from controllers.service_api import api
@@ -22,10 +23,12 @@ from core.errors.error import (
 )
 from core.model_runtime.errors.invoke import InvokeError
 from extensions.ext_database import db
+from fields.workflow_app_log_fields import workflow_app_log_pagination_fields
 from libs import helper
 from models.model import App, AppMode, EndUser
 from models.workflow import WorkflowRun
 from services.app_generate_service import AppGenerateService
+from services.workflow_app_service import WorkflowAppService
 
 logger = logging.getLogger(__name__)
 
@@ -113,6 +116,30 @@ class WorkflowTaskStopApi(Resource):
         return {"result": "success"}
 
 
+class WorkflowAppLogApi(Resource):
+    @validate_app_token
+    @marshal_with(workflow_app_log_pagination_fields)
+    def get(self, app_model: App):
+        """
+        Get workflow app logs
+        """
+        parser = reqparse.RequestParser()
+        parser.add_argument("keyword", type=str, location="args")
+        parser.add_argument("status", type=str, choices=["succeeded", "failed", "stopped"], location="args")
+        parser.add_argument("page", type=int_range(1, 99999), default=1, location="args")
+        parser.add_argument("limit", type=int_range(1, 100), default=20, location="args")
+        args = parser.parse_args()
+
+        # get paginate workflow app logs
+        workflow_app_service = WorkflowAppService()
+        workflow_app_log_pagination = workflow_app_service.get_paginate_workflow_app_logs(
+            app_model=app_model, args=args
+        )
+
+        return workflow_app_log_pagination
+
+
 api.add_resource(WorkflowRunApi, "/workflows/run")
 api.add_resource(WorkflowRunDetailApi, "/workflows/run/<string:workflow_id>")
 api.add_resource(WorkflowTaskStopApi, "/workflows/tasks/<string:task_id>/stop")
+api.add_resource(WorkflowAppLogApi, "/workflows/logs")

api/controllers/service_api/dataset/dataset.py

@@ -28,11 +28,11 @@ class DatasetListApi(DatasetApiResource):
 
         page = request.args.get("page", default=1, type=int)
         limit = request.args.get("limit", default=20, type=int)
-        provider = request.args.get("provider", default="vendor")
+        # provider = request.args.get("provider", default="vendor")
         search = request.args.get("keyword", default=None, type=str)
         tag_ids = request.args.getlist("tag_ids")
 
-        datasets, total = DatasetService.get_datasets(page, limit, provider, tenant_id, current_user, search, tag_ids)
+        datasets, total = DatasetService.get_datasets(page, limit, tenant_id, current_user, search, tag_ids)
         # check embedding setting
         provider_manager = ProviderManager()
         configurations = provider_manager.get_configurations(tenant_id=current_user.current_tenant_id)
@@ -82,6 +82,26 @@ class DatasetListApi(DatasetApiResource):
             required=False,
             nullable=False,
         )
+        parser.add_argument(
+            "external_knowledge_api_id",
+            type=str,
+            nullable=True,
+            required=False,
+            default="_validate_name",
+        )
+        parser.add_argument(
+            "provider",
+            type=str,
+            nullable=True,
+            required=False,
+            default="vendor",
+        )
+        parser.add_argument(
+            "external_knowledge_id",
+            type=str,
+            nullable=True,
+            required=False,
+        )
         args = parser.parse_args()
 
         try:
@@ -91,6 +111,9 @@ class DatasetListApi(DatasetApiResource):
                 indexing_technique=args["indexing_technique"],
                 account=current_user,
                 permission=args["permission"],
+                provider=args["provider"],
+                external_knowledge_api_id=args["external_knowledge_api_id"],
+                external_knowledge_id=args["external_knowledge_id"],
             )
         except services.errors.dataset.DatasetNameDuplicateError:
             raise DatasetNameDuplicateError()

api/controllers/service_api/dataset/segment.py

@@ -36,6 +36,10 @@ class SegmentApi(DatasetApiResource):
         document = DocumentService.get_document(dataset.id, document_id)
         if not document:
             raise NotFound("Document not found.")
+        if document.indexing_status != "completed":
+            raise NotFound("Document is not completed.")
+        if not document.enabled:
+            raise NotFound("Document is disabled.")
         # check embedding model setting
         if dataset.indexing_technique == "high_quality":
             try:
@@ -63,7 +67,7 @@ class SegmentApi(DatasetApiResource):
             segments = SegmentService.multi_create_segment(args["segments"], document, dataset)
             return {"data": marshal(segments, segment_fields), "doc_form": document.doc_form}, 200
         else:
-            return {"error": "Segemtns is required"}, 400
+            return {"error": "Segments is required"}, 400
 
     def get(self, tenant_id, dataset_id, document_id):
         """Create single segment."""

api/controllers/web/app.py

@@ -41,7 +41,7 @@ class AppParameterApi(WebApiResource):
     @marshal_with(parameters_fields)
     def get(self, app_model: App, end_user):
         """Retrieve app parameters."""
-        if app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]:
+        if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}:
             workflow = app_model.workflow
             if workflow is None:
                 raise AppUnavailableError()

api/controllers/web/audio.py

@@ -78,19 +78,15 @@ class TextApi(WebApiResource):
             message_id = args.get("message_id", None)
             text = args.get("text", None)
             if (
-                app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]
+                app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}
                 and app_model.workflow
                 and app_model.workflow.features_dict
             ):
                 text_to_speech = app_model.workflow.features_dict.get("text_to_speech")
-                voice = args.get("voice") if args.get("voice") else text_to_speech.get("voice")
+                voice = args.get("voice") or text_to_speech.get("voice")
             else:
                 try:
-                    voice = (
-                        args.get("voice")
-                        if args.get("voice")
-                        else app_model.app_model_config.text_to_speech_dict.get("voice")
-                    )
+                    voice = args.get("voice") or app_model.app_model_config.text_to_speech_dict.get("voice")
                 except Exception:
                     voice = None
 

api/controllers/web/completion.py

@@ -87,7 +87,7 @@ class CompletionStopApi(WebApiResource):
 class ChatApi(WebApiResource):
     def post(self, app_model, end_user):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -96,6 +96,7 @@ class ChatApi(WebApiResource):
         parser.add_argument("files", type=list, required=False, location="json")
         parser.add_argument("response_mode", type=str, choices=["blocking", "streaming"], location="json")
         parser.add_argument("conversation_id", type=uuid_value, location="json")
+        parser.add_argument("parent_message_id", type=uuid_value, required=False, location="json")
         parser.add_argument("retriever_from", type=str, required=False, default="web_app", location="json")
 
         args = parser.parse_args()
@@ -136,7 +137,7 @@ class ChatApi(WebApiResource):
 class ChatStopApi(WebApiResource):
     def post(self, app_model, end_user, task_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         AppQueueManager.set_stop_flag(task_id, InvokeFrom.WEB_APP, end_user.id)

api/controllers/web/conversation.py

@@ -18,7 +18,7 @@ class ConversationListApi(WebApiResource):
     @marshal_with(conversation_infinite_scroll_pagination_fields)
     def get(self, app_model, end_user):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -56,7 +56,7 @@ class ConversationListApi(WebApiResource):
 class ConversationApi(WebApiResource):
     def delete(self, app_model, end_user, c_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -73,7 +73,7 @@ class ConversationRenameApi(WebApiResource):
     @marshal_with(simple_conversation_fields)
     def post(self, app_model, end_user, c_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -92,7 +92,7 @@ class ConversationRenameApi(WebApiResource):
 class ConversationPinApi(WebApiResource):
     def patch(self, app_model, end_user, c_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -108,7 +108,7 @@ class ConversationPinApi(WebApiResource):
 class ConversationUnPinApi(WebApiResource):
     def patch(self, app_model, end_user, c_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)

api/controllers/web/message.py

@@ -57,6 +57,7 @@ class MessageListApi(WebApiResource):
     message_fields = {
         "id": fields.String,
         "conversation_id": fields.String,
+        "parent_message_id": fields.String,
         "inputs": fields.Raw,
         "query": fields.String,
         "answer": fields.String(attribute="re_sign_file_url_answer"),
@@ -78,7 +79,7 @@ class MessageListApi(WebApiResource):
     @marshal_with(message_infinite_scroll_pagination_fields)
     def get(self, app_model, end_user):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -89,7 +90,7 @@ class MessageListApi(WebApiResource):
 
         try:
             return MessageService.pagination_by_first_id(
-                app_model, end_user, args["conversation_id"], args["first_id"], args["limit"]
+                app_model, end_user, args["conversation_id"], args["first_id"], args["limit"], "desc"
             )
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
@@ -160,7 +161,7 @@ class MessageMoreLikeThisApi(WebApiResource):
 class MessageSuggestedQuestionApi(WebApiResource):
     def get(self, app_model, end_user, message_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotCompletionAppError()
 
         message_id = str(message_id)

api/controllers/web/wraps.py

@@ -80,7 +80,8 @@ def _validate_web_sso_token(decoded, system_features, app_code):
             if not source or source != "sso":
                 raise WebSSOAuthRequiredError()
 
-    # Check if SSO is not enforced for web, and if the token source is SSO, raise an error and redirect to normal passport login
+    # Check if SSO is not enforced for web, and if the token source is SSO,
+    # raise an error and redirect to normal passport login
     if not system_features.sso_enforced_for_web or not app_web_sso_enabled:
         source = decoded.get("token_source")
         if source and source == "sso":

api/core/__init__.py

@@ -1 +1 @@
-import core.moderation.base
+import core.moderation.base

api/core/agent/base_agent_runner.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import uuid
+from collections.abc import Mapping, Sequence
 from datetime import datetime, timezone
 from typing import Optional, Union, cast
 
@@ -31,6 +32,7 @@ from core.model_runtime.entities.message_entities import (
 from core.model_runtime.entities.model_entities import ModelFeature
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.model_runtime.utils.encoders import jsonable_encoder
+from core.prompt.utils.extract_thread_messages import extract_thread_messages
 from core.tools.entities.tool_entities import (
     ToolParameter,
     ToolRuntimeVariablePool,
@@ -45,22 +47,25 @@ from models.tools import ToolConversationVariables
 
 logger = logging.getLogger(__name__)
 
+
 class BaseAgentRunner(AppRunner):
-    def __init__(self, tenant_id: str,
-                 application_generate_entity: AgentChatAppGenerateEntity,
-                 conversation: Conversation,
-                 app_config: AgentChatAppConfig,
-                 model_config: ModelConfigWithCredentialsEntity,
-                 config: AgentEntity,
-                 queue_manager: AppQueueManager,
-                 message: Message,
-                 user_id: str,
-                 memory: Optional[TokenBufferMemory] = None,
-                 prompt_messages: Optional[list[PromptMessage]] = None,
-                 variables_pool: Optional[ToolRuntimeVariablePool] = None,
-                 db_variables: Optional[ToolConversationVariables] = None,
-                 model_instance: ModelInstance = None
-                 ) -> None:
+    def __init__(
+        self,
+        tenant_id: str,
+        application_generate_entity: AgentChatAppGenerateEntity,
+        conversation: Conversation,
+        app_config: AgentChatAppConfig,
+        model_config: ModelConfigWithCredentialsEntity,
+        config: AgentEntity,
+        queue_manager: AppQueueManager,
+        message: Message,
+        user_id: str,
+        memory: Optional[TokenBufferMemory] = None,
+        prompt_messages: Optional[list[PromptMessage]] = None,
+        variables_pool: Optional[ToolRuntimeVariablePool] = None,
+        db_variables: Optional[ToolConversationVariables] = None,
+        model_instance: ModelInstance = None,
+    ) -> None:
         """
         Agent runner
         :param tenant_id: tenant id
@@ -88,9 +93,7 @@ class BaseAgentRunner(AppRunner):
         self.message = message
         self.user_id = user_id
         self.memory = memory
-        self.history_prompt_messages = self.organize_agent_history(
-            prompt_messages=prompt_messages or []
-        )
+        self.history_prompt_messages = self.organize_agent_history(prompt_messages=prompt_messages or [])
         self.variables_pool = variables_pool
         self.db_variables_pool = db_variables
         self.model_instance = model_instance
@@ -111,12 +114,16 @@ class BaseAgentRunner(AppRunner):
             retrieve_config=app_config.dataset.retrieve_config if app_config.dataset else None,
             return_resource=app_config.additional_features.show_retrieve_source,
             invoke_from=application_generate_entity.invoke_from,
-            hit_callback=hit_callback
+            hit_callback=hit_callback,
         )
         # get how many agent thoughts have been created
-        self.agent_thought_count = db.session.query(MessageAgentThought).filter(
-            MessageAgentThought.message_id == self.message.id,
-        ).count()
+        self.agent_thought_count = (
+            db.session.query(MessageAgentThought)
+            .filter(
+                MessageAgentThought.message_id == self.message.id,
+            )
+            .count()
+        )
         db.session.close()
 
         # check if model supports stream tool call
@@ -135,25 +142,26 @@ class BaseAgentRunner(AppRunner):
         self.query = None
         self._current_thoughts: list[PromptMessage] = []
 
-    def _repack_app_generate_entity(self, app_generate_entity: AgentChatAppGenerateEntity) \
-            -> AgentChatAppGenerateEntity:
+    def _repack_app_generate_entity(
+        self, app_generate_entity: AgentChatAppGenerateEntity
+    ) -> AgentChatAppGenerateEntity:
         """
         Repack app generate entity
         """
         if app_generate_entity.app_config.prompt_template.simple_prompt_template is None:
-            app_generate_entity.app_config.prompt_template.simple_prompt_template = ''
+            app_generate_entity.app_config.prompt_template.simple_prompt_template = ""
 
         return app_generate_entity
-    
+
     def _convert_tool_to_prompt_message_tool(self, tool: AgentToolEntity) -> tuple[PromptMessageTool, Tool]:
         """
-            convert tool to prompt message tool
+        convert tool to prompt message tool
         """
         tool_entity = ToolManager.get_agent_tool_runtime(
             tenant_id=self.tenant_id,
             app_id=self.app_config.app_id,
             agent_tool=tool,
-            invoke_from=self.application_generate_entity.invoke_from
+            invoke_from=self.application_generate_entity.invoke_from,
         )
         tool_entity.load_variables(self.variables_pool)
 
@@ -164,7 +172,7 @@ class BaseAgentRunner(AppRunner):
                 "type": "object",
                 "properties": {},
                 "required": [],
-            }
+            },
         )
 
         parameters = tool_entity.get_all_runtime_parameters()
@@ -177,19 +185,19 @@ class BaseAgentRunner(AppRunner):
             if parameter.type == ToolParameter.ToolParameterType.SELECT:
                 enum = [option.value for option in parameter.options]
 
-            message_tool.parameters['properties'][parameter.name] = {
+            message_tool.parameters["properties"][parameter.name] = {
                 "type": parameter_type,
-                "description": parameter.llm_description or '',
+                "description": parameter.llm_description or "",
             }
 
             if len(enum) > 0:
-                message_tool.parameters['properties'][parameter.name]['enum'] = enum
+                message_tool.parameters["properties"][parameter.name]["enum"] = enum
 
             if parameter.required:
-                message_tool.parameters['required'].append(parameter.name)
+                message_tool.parameters["required"].append(parameter.name)
 
         return message_tool, tool_entity
-    
+
     def _convert_dataset_retriever_tool_to_prompt_message_tool(self, tool: DatasetRetrieverTool) -> PromptMessageTool:
         """
         convert dataset retriever tool to prompt message tool
@@ -201,24 +209,24 @@ class BaseAgentRunner(AppRunner):
                 "type": "object",
                 "properties": {},
                 "required": [],
-            }
+            },
         )
 
         for parameter in tool.get_runtime_parameters():
-            parameter_type = 'string'
-        
-            prompt_tool.parameters['properties'][parameter.name] = {
+            parameter_type = "string"
+
+            prompt_tool.parameters["properties"][parameter.name] = {
                 "type": parameter_type,
-                "description": parameter.llm_description or '',
+                "description": parameter.llm_description or "",
             }
 
             if parameter.required:
-                if parameter.name not in prompt_tool.parameters['required']:
-                    prompt_tool.parameters['required'].append(parameter.name)
+                if parameter.name not in prompt_tool.parameters["required"]:
+                    prompt_tool.parameters["required"].append(parameter.name)
 
         return prompt_tool
-    
-    def _init_prompt_tools(self) -> tuple[dict[str, Tool], list[PromptMessageTool]]:
+
+    def _init_prompt_tools(self) -> tuple[Mapping[str, Tool], Sequence[PromptMessageTool]]:
         """
         Init tools
         """
@@ -261,51 +269,51 @@ class BaseAgentRunner(AppRunner):
             enum = []
             if parameter.type == ToolParameter.ToolParameterType.SELECT:
                 enum = [option.value for option in parameter.options]
-        
-            prompt_tool.parameters['properties'][parameter.name] = {
+
+            prompt_tool.parameters["properties"][parameter.name] = {
                 "type": parameter_type,
-                "description": parameter.llm_description or '',
+                "description": parameter.llm_description or "",
             }
 
             if len(enum) > 0:
-                prompt_tool.parameters['properties'][parameter.name]['enum'] = enum
+                prompt_tool.parameters["properties"][parameter.name]["enum"] = enum
 
             if parameter.required:
-                if parameter.name not in prompt_tool.parameters['required']:
-                    prompt_tool.parameters['required'].append(parameter.name)
+                if parameter.name not in prompt_tool.parameters["required"]:
+                    prompt_tool.parameters["required"].append(parameter.name)
 
         return prompt_tool
-        
-    def create_agent_thought(self, message_id: str, message: str, 
-                             tool_name: str, tool_input: str, messages_ids: list[str]
-                             ) -> MessageAgentThought:
+
+    def create_agent_thought(
+        self, message_id: str, message: str, tool_name: str, tool_input: str, messages_ids: list[str]
+    ) -> MessageAgentThought:
         """
         Create agent thought
         """
         thought = MessageAgentThought(
             message_id=message_id,
             message_chain_id=None,
-            thought='',
+            thought="",
             tool=tool_name,
-            tool_labels_str='{}',
-            tool_meta_str='{}',
+            tool_labels_str="{}",
+            tool_meta_str="{}",
             tool_input=tool_input,
             message=message,
             message_token=0,
             message_unit_price=0,
             message_price_unit=0,
-            message_files=json.dumps(messages_ids) if messages_ids else '',
-            answer='',
-            observation='',
+            message_files=json.dumps(messages_ids) if messages_ids else "",
+            answer="",
+            observation="",
             answer_token=0,
             answer_unit_price=0,
             answer_price_unit=0,
             tokens=0,
             total_price=0,
             position=self.agent_thought_count + 1,
-            currency='USD',
+            currency="USD",
             latency=0,
-            created_by_role='account',
+            created_by_role="account",
             created_by=self.user_id,
         )
 
@@ -318,22 +326,22 @@ class BaseAgentRunner(AppRunner):
 
         return thought
 
-    def save_agent_thought(self, 
-                           agent_thought: MessageAgentThought, 
-                           tool_name: str,
-                           tool_input: Union[str, dict],
-                           thought: str, 
-                           observation: Union[str, dict], 
-                           tool_invoke_meta: Union[str, dict],
-                           answer: str,
-                           messages_ids: list[str],
-                           llm_usage: LLMUsage = None) -> MessageAgentThought:
+    def save_agent_thought(
+        self,
+        agent_thought: MessageAgentThought,
+        tool_name: str,
+        tool_input: Union[str, dict],
+        thought: str,
+        observation: Union[str, dict],
+        tool_invoke_meta: Union[str, dict],
+        answer: str,
+        messages_ids: list[str],
+        llm_usage: LLMUsage = None,
+    ) -> MessageAgentThought:
         """
         Save agent thought
         """
-        agent_thought = db.session.query(MessageAgentThought).filter(
-            MessageAgentThought.id == agent_thought.id
-        ).first()
+        agent_thought = db.session.query(MessageAgentThought).filter(MessageAgentThought.id == agent_thought.id).first()
 
         if thought is not None:
             agent_thought.thought = thought
@@ -356,7 +364,7 @@ class BaseAgentRunner(AppRunner):
                     observation = json.dumps(observation, ensure_ascii=False)
                 except Exception as e:
                     observation = json.dumps(observation)
-                    
+
             agent_thought.observation = observation
 
         if answer is not None:
@@ -364,7 +372,7 @@ class BaseAgentRunner(AppRunner):
 
         if messages_ids is not None and len(messages_ids) > 0:
             agent_thought.message_files = json.dumps(messages_ids)
-        
+
         if llm_usage:
             agent_thought.message_token = llm_usage.prompt_tokens
             agent_thought.message_price_unit = llm_usage.prompt_price_unit
@@ -377,7 +385,7 @@ class BaseAgentRunner(AppRunner):
 
         # check if tool labels is not empty
         labels = agent_thought.tool_labels or {}
-        tools = agent_thought.tool.split(';') if agent_thought.tool else []
+        tools = agent_thought.tool.split(";") if agent_thought.tool else []
         for tool in tools:
             if not tool:
                 continue
@@ -386,7 +394,7 @@ class BaseAgentRunner(AppRunner):
                 if tool_label:
                     labels[tool] = tool_label.to_dict()
                 else:
-                    labels[tool] = {'en_US': tool, 'zh_Hans': tool}
+                    labels[tool] = {"en_US": tool, "zh_Hans": tool}
 
         agent_thought.tool_labels_str = json.dumps(labels)
 
@@ -401,14 +409,18 @@ class BaseAgentRunner(AppRunner):
 
         db.session.commit()
         db.session.close()
-    
+
     def update_db_variables(self, tool_variables: ToolRuntimeVariablePool, db_variables: ToolConversationVariables):
         """
         convert tool variables to db variables
         """
-        db_variables = db.session.query(ToolConversationVariables).filter(
-            ToolConversationVariables.conversation_id == self.message.conversation_id,
-        ).first()
+        db_variables = (
+            db.session.query(ToolConversationVariables)
+            .filter(
+                ToolConversationVariables.conversation_id == self.message.conversation_id,
+            )
+            .first()
+        )
 
         db_variables.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
         db_variables.variables_str = json.dumps(jsonable_encoder(tool_variables.pool))
@@ -425,9 +437,16 @@ class BaseAgentRunner(AppRunner):
             if isinstance(prompt_message, SystemPromptMessage):
                 result.append(prompt_message)
 
-        messages: list[Message] = db.session.query(Message).filter(
-            Message.conversation_id == self.message.conversation_id,
-        ).order_by(Message.created_at.asc()).all()
+        messages: list[Message] = (
+            db.session.query(Message)
+            .filter(
+                Message.conversation_id == self.message.conversation_id,
+            )
+            .order_by(Message.created_at.desc())
+            .all()
+        )
+
+        messages = list(reversed(extract_thread_messages(messages)))
 
         for message in messages:
             if message.id == self.message.id:
@@ -439,13 +458,13 @@ class BaseAgentRunner(AppRunner):
                 for agent_thought in agent_thoughts:
                     tools = agent_thought.tool
                     if tools:
-                        tools = tools.split(';')
+                        tools = tools.split(";")
                         tool_calls: list[AssistantPromptMessage.ToolCall] = []
                         tool_call_response: list[ToolPromptMessage] = []
                         try:
                             tool_inputs = json.loads(agent_thought.tool_input)
                         except Exception as e:
-                            tool_inputs = { tool: {} for tool in tools }
+                            tool_inputs = {tool: {} for tool in tools}
                         try:
                             tool_responses = json.loads(agent_thought.observation)
                         except Exception as e:
@@ -454,27 +473,33 @@ class BaseAgentRunner(AppRunner):
                         for tool in tools:
                             # generate a uuid for tool call
                             tool_call_id = str(uuid.uuid4())
-                            tool_calls.append(AssistantPromptMessage.ToolCall(
-                                id=tool_call_id,
-                                type='function',
-                                function=AssistantPromptMessage.ToolCall.ToolCallFunction(
-                                    name=tool,
-                                    arguments=json.dumps(tool_inputs.get(tool, {})),
+                            tool_calls.append(
+                                AssistantPromptMessage.ToolCall(
+                                    id=tool_call_id,
+                                    type="function",
+                                    function=AssistantPromptMessage.ToolCall.ToolCallFunction(
+                                        name=tool,
+                                        arguments=json.dumps(tool_inputs.get(tool, {})),
+                                    ),
                                 )
-                            ))
-                            tool_call_response.append(ToolPromptMessage(
-                                content=tool_responses.get(tool, agent_thought.observation),
-                                name=tool,
-                                tool_call_id=tool_call_id,
-                            ))
+                            )
+                            tool_call_response.append(
+                                ToolPromptMessage(
+                                    content=tool_responses.get(tool, agent_thought.observation),
+                                    name=tool,
+                                    tool_call_id=tool_call_id,
+                                )
+                            )
 
-                        result.extend([
-                            AssistantPromptMessage(
-                                content=agent_thought.thought,
-                                tool_calls=tool_calls,
-                            ),
-                            *tool_call_response
-                        ])
+                        result.extend(
+                            [
+                                AssistantPromptMessage(
+                                    content=agent_thought.thought,
+                                    tool_calls=tool_calls,
+                                ),
+                                *tool_call_response,
+                            ]
+                        )
                     if not tools:
                         result.append(AssistantPromptMessage(content=agent_thought.thought))
             else:
@@ -496,10 +521,7 @@ class BaseAgentRunner(AppRunner):
             file_extra_config = FileUploadConfigManager.convert(message.app_model_config.to_dict())
 
             if file_extra_config:
-                file_objs = message_file_parser.transform_message_files(
-                    files,
-                    file_extra_config
-                )
+                file_objs = message_file_parser.transform_message_files(files, file_extra_config)
             else:
                 file_objs = []
 

api/core/agent/cot_agent_runner.py

@@ -25,17 +25,19 @@ from models.model import Message
 
 class CotAgentRunner(BaseAgentRunner, ABC):
     _is_first_iteration = True
-    _ignore_observation_providers = ['wenxin']
+    _ignore_observation_providers = ["wenxin"]
     _historic_prompt_messages: list[PromptMessage] = None
     _agent_scratchpad: list[AgentScratchpadUnit] = None
     _instruction: str = None
     _query: str = None
     _prompt_messages_tools: list[PromptMessage] = None
 
-    def run(self, message: Message,
-            query: str,
-            inputs: dict[str, str],
-            ) -> Union[Generator, LLMResult]:
+    def run(
+        self,
+        message: Message,
+        query: str,
+        inputs: dict[str, str],
+    ) -> Union[Generator, LLMResult]:
         """
         Run Cot agent application
         """
@@ -46,17 +48,16 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         trace_manager = app_generate_entity.trace_manager
 
         # check model mode
-        if 'Observation' not in app_generate_entity.model_conf.stop:
+        if "Observation" not in app_generate_entity.model_conf.stop:
             if app_generate_entity.model_conf.provider not in self._ignore_observation_providers:
-                app_generate_entity.model_conf.stop.append('Observation')
+                app_generate_entity.model_conf.stop.append("Observation")
 
         app_config = self.app_config
 
         # init instruction
         inputs = inputs or {}
         instruction = app_config.prompt_template.simple_prompt_template
-        self._instruction = self._fill_in_inputs_from_external_data_tools(
-            instruction, inputs)
+        self._instruction = self._fill_in_inputs_from_external_data_tools(instruction, inputs)
 
         iteration_step = 1
         max_iteration_steps = min(app_config.agent.max_iteration, 5) + 1
@@ -65,16 +66,14 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         tool_instances, self._prompt_messages_tools = self._init_prompt_tools()
 
         function_call_state = True
-        llm_usage = {
-            'usage': None
-        }
-        final_answer = ''
+        llm_usage = {"usage": None}
+        final_answer = ""
 
         def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage):
-            if not final_llm_usage_dict['usage']:
-                final_llm_usage_dict['usage'] = usage
+            if not final_llm_usage_dict["usage"]:
+                final_llm_usage_dict["usage"] = usage
             else:
-                llm_usage = final_llm_usage_dict['usage']
+                llm_usage = final_llm_usage_dict["usage"]
                 llm_usage.prompt_tokens += usage.prompt_tokens
                 llm_usage.completion_tokens += usage.completion_tokens
                 llm_usage.prompt_price += usage.prompt_price
@@ -94,17 +93,13 @@ class CotAgentRunner(BaseAgentRunner, ABC):
             message_file_ids = []
 
             agent_thought = self.create_agent_thought(
-                message_id=message.id,
-                message='',
-                tool_name='',
-                tool_input='',
-                messages_ids=message_file_ids
+                message_id=message.id, message="", tool_name="", tool_input="", messages_ids=message_file_ids
             )
 
             if iteration_step > 1:
-                self.queue_manager.publish(QueueAgentThoughtEvent(
-                    agent_thought_id=agent_thought.id
-                ), PublishFrom.APPLICATION_MANAGER)
+                self.queue_manager.publish(
+                    QueueAgentThoughtEvent(agent_thought_id=agent_thought.id), PublishFrom.APPLICATION_MANAGER
+                )
 
             # recalc llm max tokens
             prompt_messages = self._organize_prompt_messages()
@@ -125,21 +120,20 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                 raise ValueError("failed to invoke llm")
 
             usage_dict = {}
-            react_chunks = CotAgentOutputParser.handle_react_stream_output(
-                chunks, usage_dict)
+            react_chunks = CotAgentOutputParser.handle_react_stream_output(chunks, usage_dict)
             scratchpad = AgentScratchpadUnit(
-                agent_response='',
-                thought='',
-                action_str='',
-                observation='',
+                agent_response="",
+                thought="",
+                action_str="",
+                observation="",
                 action=None,
             )
 
             # publish agent thought if it's first iteration
             if iteration_step == 1:
-                self.queue_manager.publish(QueueAgentThoughtEvent(
-                    agent_thought_id=agent_thought.id
-                ), PublishFrom.APPLICATION_MANAGER)
+                self.queue_manager.publish(
+                    QueueAgentThoughtEvent(agent_thought_id=agent_thought.id), PublishFrom.APPLICATION_MANAGER
+                )
 
             for chunk in react_chunks:
                 if isinstance(chunk, AgentScratchpadUnit.Action):
@@ -154,61 +148,51 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                     yield LLMResultChunk(
                         model=self.model_config.model,
                         prompt_messages=prompt_messages,
-                        system_fingerprint='',
-                        delta=LLMResultChunkDelta(
-                            index=0,
-                            message=AssistantPromptMessage(
-                                content=chunk
-                            ),
-                            usage=None
-                        )
+                        system_fingerprint="",
+                        delta=LLMResultChunkDelta(index=0, message=AssistantPromptMessage(content=chunk), usage=None),
                     )
 
-            scratchpad.thought = scratchpad.thought.strip(
-            ) or 'I am thinking about how to help you'
+            scratchpad.thought = scratchpad.thought.strip() or "I am thinking about how to help you"
             self._agent_scratchpad.append(scratchpad)
 
             # get llm usage
-            if 'usage' in usage_dict:
-                increase_usage(llm_usage, usage_dict['usage'])
+            if "usage" in usage_dict:
+                increase_usage(llm_usage, usage_dict["usage"])
             else:
-                usage_dict['usage'] = LLMUsage.empty_usage()
+                usage_dict["usage"] = LLMUsage.empty_usage()
 
             self.save_agent_thought(
                 agent_thought=agent_thought,
-                tool_name=scratchpad.action.action_name if scratchpad.action else '',
-                tool_input={
-                    scratchpad.action.action_name: scratchpad.action.action_input
-                } if scratchpad.action else {},
+                tool_name=scratchpad.action.action_name if scratchpad.action else "",
+                tool_input={scratchpad.action.action_name: scratchpad.action.action_input} if scratchpad.action else {},
                 tool_invoke_meta={},
                 thought=scratchpad.thought,
-                observation='',
+                observation="",
                 answer=scratchpad.agent_response,
                 messages_ids=[],
-                llm_usage=usage_dict['usage']
+                llm_usage=usage_dict["usage"],
             )
 
             if not scratchpad.is_final():
-                self.queue_manager.publish(QueueAgentThoughtEvent(
-                    agent_thought_id=agent_thought.id
-                ), PublishFrom.APPLICATION_MANAGER)
+                self.queue_manager.publish(
+                    QueueAgentThoughtEvent(agent_thought_id=agent_thought.id), PublishFrom.APPLICATION_MANAGER
+                )
 
             if not scratchpad.action:
                 # failed to extract action, return final answer directly
-                final_answer = ''
+                final_answer = ""
             else:
                 if scratchpad.action.action_name.lower() == "final answer":
                     # action is final answer, return final answer directly
                     try:
                         if isinstance(scratchpad.action.action_input, dict):
-                            final_answer = json.dumps(
-                                scratchpad.action.action_input)
+                            final_answer = json.dumps(scratchpad.action.action_input)
                         elif isinstance(scratchpad.action.action_input, str):
                             final_answer = scratchpad.action.action_input
                         else:
-                            final_answer = f'{scratchpad.action.action_input}'
+                            final_answer = f"{scratchpad.action.action_input}"
                     except json.JSONDecodeError:
-                        final_answer = f'{scratchpad.action.action_input}'
+                        final_answer = f"{scratchpad.action.action_input}"
                 else:
                     function_call_state = True
                     # action is tool call, invoke tool
@@ -224,21 +208,18 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                     self.save_agent_thought(
                         agent_thought=agent_thought,
                         tool_name=scratchpad.action.action_name,
-                        tool_input={
-                            scratchpad.action.action_name: scratchpad.action.action_input},
+                        tool_input={scratchpad.action.action_name: scratchpad.action.action_input},
                         thought=scratchpad.thought,
-                        observation={
-                            scratchpad.action.action_name: tool_invoke_response},
-                        tool_invoke_meta={
-                            scratchpad.action.action_name: tool_invoke_meta.to_dict()},
+                        observation={scratchpad.action.action_name: tool_invoke_response},
+                        tool_invoke_meta={scratchpad.action.action_name: tool_invoke_meta.to_dict()},
                         answer=scratchpad.agent_response,
                         messages_ids=message_file_ids,
-                        llm_usage=usage_dict['usage']
+                        llm_usage=usage_dict["usage"],
                     )
 
-                    self.queue_manager.publish(QueueAgentThoughtEvent(
-                        agent_thought_id=agent_thought.id
-                    ), PublishFrom.APPLICATION_MANAGER)
+                    self.queue_manager.publish(
+                        QueueAgentThoughtEvent(agent_thought_id=agent_thought.id), PublishFrom.APPLICATION_MANAGER
+                    )
 
                 # update prompt tool message
                 for prompt_tool in self._prompt_messages_tools:
@@ -250,44 +231,45 @@ class CotAgentRunner(BaseAgentRunner, ABC):
             model=model_instance.model,
             prompt_messages=prompt_messages,
             delta=LLMResultChunkDelta(
-                index=0,
-                message=AssistantPromptMessage(
-                    content=final_answer
-                ),
-                usage=llm_usage['usage']
+                index=0, message=AssistantPromptMessage(content=final_answer), usage=llm_usage["usage"]
             ),
-            system_fingerprint=''
+            system_fingerprint="",
         )
 
         # save agent thought
         self.save_agent_thought(
             agent_thought=agent_thought,
-            tool_name='',
+            tool_name="",
             tool_input={},
             tool_invoke_meta={},
             thought=final_answer,
             observation={},
             answer=final_answer,
-            messages_ids=[]
+            messages_ids=[],
         )
 
         self.update_db_variables(self.variables_pool, self.db_variables_pool)
         # publish end event
-        self.queue_manager.publish(QueueMessageEndEvent(llm_result=LLMResult(
-            model=model_instance.model,
-            prompt_messages=prompt_messages,
-            message=AssistantPromptMessage(
-                content=final_answer
+        self.queue_manager.publish(
+            QueueMessageEndEvent(
+                llm_result=LLMResult(
+                    model=model_instance.model,
+                    prompt_messages=prompt_messages,
+                    message=AssistantPromptMessage(content=final_answer),
+                    usage=llm_usage["usage"] or LLMUsage.empty_usage(),
+                    system_fingerprint="",
+                )
             ),
-            usage=llm_usage['usage'] if llm_usage['usage'] else LLMUsage.empty_usage(),
-            system_fingerprint=''
-        )), PublishFrom.APPLICATION_MANAGER)
+            PublishFrom.APPLICATION_MANAGER,
+        )
 
-    def _handle_invoke_action(self, action: AgentScratchpadUnit.Action,
-                              tool_instances: dict[str, Tool],
-                              message_file_ids: list[str],
-                              trace_manager: Optional[TraceQueueManager] = None
-                              ) -> tuple[str, ToolInvokeMeta]:
+    def _handle_invoke_action(
+        self,
+        action: AgentScratchpadUnit.Action,
+        tool_instances: dict[str, Tool],
+        message_file_ids: list[str],
+        trace_manager: Optional[TraceQueueManager] = None,
+    ) -> tuple[str, ToolInvokeMeta]:
         """
         handle invoke action
         :param action: action
@@ -326,13 +308,12 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         # publish files
         for message_file_id, save_as in message_files:
             if save_as:
-                self.variables_pool.set_file(
-                    tool_name=tool_call_name, value=message_file_id, name=save_as)
+                self.variables_pool.set_file(tool_name=tool_call_name, value=message_file_id, name=save_as)
 
             # publish message file
-            self.queue_manager.publish(QueueMessageFileEvent(
-                message_file_id=message_file_id
-            ), PublishFrom.APPLICATION_MANAGER)
+            self.queue_manager.publish(
+                QueueMessageFileEvent(message_file_id=message_file_id), PublishFrom.APPLICATION_MANAGER
+            )
             # add message file ids
             message_file_ids.append(message_file_id)
 
@@ -342,10 +323,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         """
         convert dict to action
         """
-        return AgentScratchpadUnit.Action(
-            action_name=action['action'],
-            action_input=action['action_input']
-        )
+        return AgentScratchpadUnit.Action(action_name=action["action"], action_input=action["action_input"])
 
     def _fill_in_inputs_from_external_data_tools(self, instruction: str, inputs: dict) -> str:
         """
@@ -353,7 +331,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         """
         for key, value in inputs.items():
             try:
-                instruction = instruction.replace(f'{{{{{key}}}}}', str(value))
+                instruction = instruction.replace(f"{{{{{key}}}}}", str(value))
             except Exception as e:
                 continue
 
@@ -370,14 +348,14 @@ class CotAgentRunner(BaseAgentRunner, ABC):
     @abstractmethod
     def _organize_prompt_messages(self) -> list[PromptMessage]:
         """
-            organize prompt messages
+        organize prompt messages
         """
 
     def _format_assistant_message(self, agent_scratchpad: list[AgentScratchpadUnit]) -> str:
         """
-            format assistant message
+        format assistant message
         """
-        message = ''
+        message = ""
         for scratchpad in agent_scratchpad:
             if scratchpad.is_final():
                 message += f"Final Answer: {scratchpad.agent_response}"
@@ -390,9 +368,11 @@ class CotAgentRunner(BaseAgentRunner, ABC):
 
         return message
 
-    def _organize_historic_prompt_messages(self, current_session_messages: list[PromptMessage] = None) -> list[PromptMessage]:
+    def _organize_historic_prompt_messages(
+        self, current_session_messages: list[PromptMessage] = None
+    ) -> list[PromptMessage]:
         """
-            organize historic prompt messages
+        organize historic prompt messages
         """
         result: list[PromptMessage] = []
         scratchpads: list[AgentScratchpadUnit] = []
@@ -403,8 +383,8 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                 if not current_scratchpad:
                     current_scratchpad = AgentScratchpadUnit(
                         agent_response=message.content,
-                        thought=message.content or 'I am thinking about how to help you',
-                        action_str='',
+                        thought=message.content or "I am thinking about how to help you",
+                        action_str="",
                         action=None,
                         observation=None,
                     )
@@ -413,12 +393,9 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                     try:
                         current_scratchpad.action = AgentScratchpadUnit.Action(
                             action_name=message.tool_calls[0].function.name,
-                            action_input=json.loads(
-                                message.tool_calls[0].function.arguments)
-                        )
-                        current_scratchpad.action_str = json.dumps(
-                            current_scratchpad.action.to_dict()
+                            action_input=json.loads(message.tool_calls[0].function.arguments),
                         )
+                        current_scratchpad.action_str = json.dumps(current_scratchpad.action.to_dict())
                     except:
                         pass
             elif isinstance(message, ToolPromptMessage):
@@ -426,23 +403,19 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                     current_scratchpad.observation = message.content
             elif isinstance(message, UserPromptMessage):
                 if scratchpads:
-                    result.append(AssistantPromptMessage(
-                        content=self._format_assistant_message(scratchpads)
-                    ))
+                    result.append(AssistantPromptMessage(content=self._format_assistant_message(scratchpads)))
                     scratchpads = []
                     current_scratchpad = None
 
                 result.append(message)
 
         if scratchpads:
-            result.append(AssistantPromptMessage(
-                content=self._format_assistant_message(scratchpads)
-            ))
+            result.append(AssistantPromptMessage(content=self._format_assistant_message(scratchpads)))
 
         historic_prompts = AgentHistoryPromptTransform(
             model_config=self.model_config,
             prompt_messages=current_session_messages or [],
             history_messages=result,
-            memory=self.memory
+            memory=self.memory,
         ).get_prompt()
         return historic_prompts

api/core/agent/cot_chat_agent_runner.py

@@ -19,14 +19,15 @@ class CotChatAgentRunner(CotAgentRunner):
         prompt_entity = self.app_config.agent.prompt
         first_prompt = prompt_entity.first_prompt
 
-        system_prompt = first_prompt \
-            .replace("{{instruction}}", self._instruction) \
-            .replace("{{tools}}", json.dumps(jsonable_encoder(self._prompt_messages_tools))) \
-            .replace("{{tool_names}}", ', '.join([tool.name for tool in self._prompt_messages_tools]))
+        system_prompt = (
+            first_prompt.replace("{{instruction}}", self._instruction)
+            .replace("{{tools}}", json.dumps(jsonable_encoder(self._prompt_messages_tools)))
+            .replace("{{tool_names}}", ", ".join([tool.name for tool in self._prompt_messages_tools]))
+        )
 
         return SystemPromptMessage(content=system_prompt)
 
-    def _organize_user_query(self, query,  prompt_messages: list[PromptMessage] = None) -> list[PromptMessage]:
+    def _organize_user_query(self, query, prompt_messages: list[PromptMessage] = None) -> list[PromptMessage]:
         """
         Organize user query
         """
@@ -43,7 +44,7 @@ class CotChatAgentRunner(CotAgentRunner):
 
     def _organize_prompt_messages(self) -> list[PromptMessage]:
         """
-        Organize 
+        Organize
         """
         # organize system prompt
         system_message = self._organize_system_prompt()
@@ -53,7 +54,7 @@ class CotChatAgentRunner(CotAgentRunner):
         if not agent_scratchpad:
             assistant_messages = []
         else:
-            assistant_message = AssistantPromptMessage(content='')
+            assistant_message = AssistantPromptMessage(content="")
             for unit in agent_scratchpad:
                 if unit.is_final():
                     assistant_message.content += f"Final Answer: {unit.agent_response}"
@@ -71,18 +72,15 @@ class CotChatAgentRunner(CotAgentRunner):
 
         if assistant_messages:
             # organize historic prompt messages
-            historic_messages = self._organize_historic_prompt_messages([
-                system_message,
-                *query_messages,
-                *assistant_messages,
-                UserPromptMessage(content='continue')
-            ])
+            historic_messages = self._organize_historic_prompt_messages(
+                [system_message, *query_messages, *assistant_messages, UserPromptMessage(content="continue")]
+            )
             messages = [
                 system_message,
                 *historic_messages,
                 *query_messages,
                 *assistant_messages,
-                UserPromptMessage(content='continue')
+                UserPromptMessage(content="continue"),
             ]
         else:
             # organize historic prompt messages

api/core/agent/cot_completion_agent_runner.py

@@ -13,10 +13,12 @@ class CotCompletionAgentRunner(CotAgentRunner):
         prompt_entity = self.app_config.agent.prompt
         first_prompt = prompt_entity.first_prompt
 
-        system_prompt = first_prompt.replace("{{instruction}}", self._instruction) \
-            .replace("{{tools}}", json.dumps(jsonable_encoder(self._prompt_messages_tools))) \
-            .replace("{{tool_names}}", ', '.join([tool.name for tool in self._prompt_messages_tools]))
-        
+        system_prompt = (
+            first_prompt.replace("{{instruction}}", self._instruction)
+            .replace("{{tools}}", json.dumps(jsonable_encoder(self._prompt_messages_tools)))
+            .replace("{{tool_names}}", ", ".join([tool.name for tool in self._prompt_messages_tools]))
+        )
+
         return system_prompt
 
     def _organize_historic_prompt(self, current_session_messages: list[PromptMessage] = None) -> str:
@@ -46,7 +48,7 @@ class CotCompletionAgentRunner(CotAgentRunner):
 
         # organize current assistant messages
         agent_scratchpad = self._agent_scratchpad
-        assistant_prompt = ''
+        assistant_prompt = ""
         for unit in agent_scratchpad:
             if unit.is_final():
                 assistant_prompt += f"Final Answer: {unit.agent_response}"
@@ -61,9 +63,10 @@ class CotCompletionAgentRunner(CotAgentRunner):
         query_prompt = f"Question: {self._query}"
 
         # join all messages
-        prompt = system_prompt \
-            .replace("{{historic_messages}}", historic_prompt) \
-            .replace("{{agent_scratchpad}}", assistant_prompt) \
+        prompt = (
+            system_prompt.replace("{{historic_messages}}", historic_prompt)
+            .replace("{{agent_scratchpad}}", assistant_prompt)
             .replace("{{query}}", query_prompt)
+        )
 
-        return [UserPromptMessage(content=prompt)]
+        return [UserPromptMessage(content=prompt)]

api/core/agent/entities.py

@@ -8,6 +8,7 @@ class AgentToolEntity(BaseModel):
     """
     Agent Tool Entity.
     """
+
     provider_type: Literal["builtin", "api", "workflow"]
     provider_id: str
     tool_name: str
@@ -18,6 +19,7 @@ class AgentPromptEntity(BaseModel):
     """
     Agent Prompt Entity.
     """
+
     first_prompt: str
     next_iteration: str
 
@@ -31,6 +33,7 @@ class AgentScratchpadUnit(BaseModel):
         """
         Action Entity.
         """
+
         action_name: str
         action_input: Union[dict, str]
 
@@ -39,8 +42,8 @@ class AgentScratchpadUnit(BaseModel):
             Convert to dictionary.
             """
             return {
-                'action': self.action_name,
-                'action_input': self.action_input,
+                "action": self.action_name,
+                "action_input": self.action_input,
             }
 
     agent_response: Optional[str] = None
@@ -54,10 +57,10 @@ class AgentScratchpadUnit(BaseModel):
         Check if the scratchpad unit is final.
         """
         return self.action is None or (
-            'final' in self.action.action_name.lower() and 
-            'answer' in self.action.action_name.lower()
+            "final" in self.action.action_name.lower() and "answer" in self.action.action_name.lower()
         )
 
+
 class AgentEntity(BaseModel):
     """
     Agent Entity.
@@ -67,8 +70,9 @@ class AgentEntity(BaseModel):
         """
         Agent Strategy.
         """
-        CHAIN_OF_THOUGHT = 'chain-of-thought'
-        FUNCTION_CALLING = 'function-calling'
+
+        CHAIN_OF_THOUGHT = "chain-of-thought"
+        FUNCTION_CALLING = "function-calling"
 
     provider: str
     model: str

api/core/agent/fc_agent_runner.py

@@ -24,11 +24,9 @@ from models.model import Message
 
 logger = logging.getLogger(__name__)
 
-class FunctionCallAgentRunner(BaseAgentRunner):
 
-    def run(self, 
-            message: Message, query: str, **kwargs: Any
-    ) -> Generator[LLMResultChunk, None, None]:
+class FunctionCallAgentRunner(BaseAgentRunner):
+    def run(self, message: Message, query: str, **kwargs: Any) -> Generator[LLMResultChunk, None, None]:
         """
         Run FunctionCall agent application
         """
@@ -45,19 +43,17 @@ class FunctionCallAgentRunner(BaseAgentRunner):
 
         # continue to run until there is not any tool call
         function_call_state = True
-        llm_usage = {
-            'usage': None
-        }
-        final_answer = ''
+        llm_usage = {"usage": None}
+        final_answer = ""
 
         # get tracing instance
         trace_manager = app_generate_entity.trace_manager
-        
+
         def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage):
-            if not final_llm_usage_dict['usage']:
-                final_llm_usage_dict['usage'] = usage
+            if not final_llm_usage_dict["usage"]:
+                final_llm_usage_dict["usage"] = usage
             else:
-                llm_usage = final_llm_usage_dict['usage']
+                llm_usage = final_llm_usage_dict["usage"]
                 llm_usage.prompt_tokens += usage.prompt_tokens
                 llm_usage.completion_tokens += usage.completion_tokens
                 llm_usage.prompt_price += usage.prompt_price
@@ -75,11 +71,7 @@ class FunctionCallAgentRunner(BaseAgentRunner):
 
             message_file_ids = []
             agent_thought = self.create_agent_thought(
-                message_id=message.id,
-                message='',
-                tool_name='',
-                tool_input='',
-                messages_ids=message_file_ids
+                message_id=message.id, message="", tool_name="", tool_input="", messages_ids=message_file_ids
             )
 
             # recalc llm max tokens
@@ -99,11 +91,11 @@ class FunctionCallAgentRunner(BaseAgentRunner):
             tool_calls: list[tuple[str, str, dict[str, Any]]] = []
 
             # save full response
-            response = ''
+            response = ""
 
             # save tool call names and inputs
-            tool_call_names = ''
-            tool_call_inputs = ''
+            tool_call_names = ""
+            tool_call_inputs = ""
 
             current_llm_usage = None
 
@@ -111,24 +103,22 @@ class FunctionCallAgentRunner(BaseAgentRunner):
                 is_first_chunk = True
                 for chunk in chunks:
                     if is_first_chunk:
-                        self.queue_manager.publish(QueueAgentThoughtEvent(
-                            agent_thought_id=agent_thought.id
-                        ), PublishFrom.APPLICATION_MANAGER)
+                        self.queue_manager.publish(
+                            QueueAgentThoughtEvent(agent_thought_id=agent_thought.id), PublishFrom.APPLICATION_MANAGER
+                        )
                         is_first_chunk = False
                     # check if there is any tool call
                     if self.check_tool_calls(chunk):
                         function_call_state = True
                         tool_calls.extend(self.extract_tool_calls(chunk))
-                        tool_call_names = ';'.join([tool_call[1] for tool_call in tool_calls])
+                        tool_call_names = ";".join([tool_call[1] for tool_call in tool_calls])
                         try:
-                            tool_call_inputs = json.dumps({
-                                tool_call[1]: tool_call[2] for tool_call in tool_calls
-                            }, ensure_ascii=False)
+                            tool_call_inputs = json.dumps(
+                                {tool_call[1]: tool_call[2] for tool_call in tool_calls}, ensure_ascii=False
+                            )
                         except json.JSONDecodeError as e:
                             # ensure ascii to avoid encoding error
-                            tool_call_inputs = json.dumps({
-                                tool_call[1]: tool_call[2] for tool_call in tool_calls
-                            })
+                            tool_call_inputs = json.dumps({tool_call[1]: tool_call[2] for tool_call in tool_calls})
 
                     if chunk.delta.message and chunk.delta.message.content:
                         if isinstance(chunk.delta.message.content, list):
@@ -148,16 +138,14 @@ class FunctionCallAgentRunner(BaseAgentRunner):
                 if self.check_blocking_tool_calls(result):
                     function_call_state = True
                     tool_calls.extend(self.extract_blocking_tool_calls(result))
-                    tool_call_names = ';'.join([tool_call[1] for tool_call in tool_calls])
+                    tool_call_names = ";".join([tool_call[1] for tool_call in tool_calls])
                     try:
-                        tool_call_inputs = json.dumps({
-                            tool_call[1]: tool_call[2] for tool_call in tool_calls
-                        }, ensure_ascii=False)
+                        tool_call_inputs = json.dumps(
+                            {tool_call[1]: tool_call[2] for tool_call in tool_calls}, ensure_ascii=False
+                        )
                     except json.JSONDecodeError as e:
                         # ensure ascii to avoid encoding error
-                        tool_call_inputs = json.dumps({
-                            tool_call[1]: tool_call[2] for tool_call in tool_calls
-                        })
+                        tool_call_inputs = json.dumps({tool_call[1]: tool_call[2] for tool_call in tool_calls})
 
                 if result.usage:
                     increase_usage(llm_usage, result.usage)
@@ -171,12 +159,12 @@ class FunctionCallAgentRunner(BaseAgentRunner):
                         response += result.message.content
 
                 if not result.message.content:
-                    result.message.content = ''
+                    result.message.content = ""
+
+                self.queue_manager.publish(
+                    QueueAgentThoughtEvent(agent_thought_id=agent_thought.id), PublishFrom.APPLICATION_MANAGER
+                )
 
-                self.queue_manager.publish(QueueAgentThoughtEvent(
-                    agent_thought_id=agent_thought.id
-                ), PublishFrom.APPLICATION_MANAGER)
-                
                 yield LLMResultChunk(
                     model=model_instance.model,
                     prompt_messages=result.prompt_messages,
@@ -185,32 +173,29 @@ class FunctionCallAgentRunner(BaseAgentRunner):
                         index=0,
                         message=result.message,
                         usage=result.usage,
-                    )
+                    ),
                 )
 
-            assistant_message = AssistantPromptMessage(
-                content='',
-                tool_calls=[]
-            )
+            assistant_message = AssistantPromptMessage(content="", tool_calls=[])
             if tool_calls:
-                assistant_message.tool_calls=[
+                assistant_message.tool_calls = [
                     AssistantPromptMessage.ToolCall(
                         id=tool_call[0],
-                        type='function',
+                        type="function",
                         function=AssistantPromptMessage.ToolCall.ToolCallFunction(
-                            name=tool_call[1],
-                            arguments=json.dumps(tool_call[2], ensure_ascii=False)
-                        )
-                    ) for tool_call in tool_calls
+                            name=tool_call[1], arguments=json.dumps(tool_call[2], ensure_ascii=False)
+                        ),
+                    )
+                    for tool_call in tool_calls
                 ]
             else:
                 assistant_message.content = response
-            
+
             self._current_thoughts.append(assistant_message)
 
             # save thought
             self.save_agent_thought(
-                agent_thought=agent_thought, 
+                agent_thought=agent_thought,
                 tool_name=tool_call_names,
                 tool_input=tool_call_inputs,
                 thought=response,
@@ -218,13 +203,13 @@ class FunctionCallAgentRunner(BaseAgentRunner):
                 observation=None,
                 answer=response,
                 messages_ids=[],
-                llm_usage=current_llm_usage
+                llm_usage=current_llm_usage,
             )
-            self.queue_manager.publish(QueueAgentThoughtEvent(
-                agent_thought_id=agent_thought.id
-            ), PublishFrom.APPLICATION_MANAGER)
-            
-            final_answer += response + '\n'
+            self.queue_manager.publish(
+                QueueAgentThoughtEvent(agent_thought_id=agent_thought.id), PublishFrom.APPLICATION_MANAGER
+            )
+
+            final_answer += response + "\n"
 
             # call tools
             tool_responses = []
@@ -235,7 +220,7 @@ class FunctionCallAgentRunner(BaseAgentRunner):
                         "tool_call_id": tool_call_id,
                         "tool_call_name": tool_call_name,
                         "tool_response": f"there is not a tool named {tool_call_name}",
-                        "meta": ToolInvokeMeta.error_instance(f"there is not a tool named {tool_call_name}").to_dict()
+                        "meta": ToolInvokeMeta.error_instance(f"there is not a tool named {tool_call_name}").to_dict(),
                     }
                 else:
                     # invoke tool
@@ -255,50 +240,49 @@ class FunctionCallAgentRunner(BaseAgentRunner):
                             self.variables_pool.set_file(tool_name=tool_call_name, value=message_file_id, name=save_as)
 
                         # publish message file
-                        self.queue_manager.publish(QueueMessageFileEvent(
-                            message_file_id=message_file_id
-                        ), PublishFrom.APPLICATION_MANAGER)
+                        self.queue_manager.publish(
+                            QueueMessageFileEvent(message_file_id=message_file_id), PublishFrom.APPLICATION_MANAGER
+                        )
                         # add message file ids
                         message_file_ids.append(message_file_id)
-                    
+
                     tool_response = {
                         "tool_call_id": tool_call_id,
                         "tool_call_name": tool_call_name,
                         "tool_response": tool_invoke_response,
-                        "meta": tool_invoke_meta.to_dict()
+                        "meta": tool_invoke_meta.to_dict(),
                     }
-                
+
                 tool_responses.append(tool_response)
-                if tool_response['tool_response'] is not None:
+                if tool_response["tool_response"] is not None:
                     self._current_thoughts.append(
                         ToolPromptMessage(
-                            content=tool_response['tool_response'],
+                            content=tool_response["tool_response"],
                             tool_call_id=tool_call_id,
                             name=tool_call_name,
                         )
-                    ) 
+                    )
 
             if len(tool_responses) > 0:
                 # save agent thought
                 self.save_agent_thought(
-                    agent_thought=agent_thought, 
+                    agent_thought=agent_thought,
                     tool_name=None,
                     tool_input=None,
-                    thought=None, 
+                    thought=None,
                     tool_invoke_meta={
-                        tool_response['tool_call_name']: tool_response['meta'] 
-                        for tool_response in tool_responses
+                        tool_response["tool_call_name"]: tool_response["meta"] for tool_response in tool_responses
                     },
                     observation={
-                        tool_response['tool_call_name']: tool_response['tool_response'] 
+                        tool_response["tool_call_name"]: tool_response["tool_response"]
                         for tool_response in tool_responses
                     },
                     answer=None,
-                    messages_ids=message_file_ids
+                    messages_ids=message_file_ids,
+                )
+                self.queue_manager.publish(
+                    QueueAgentThoughtEvent(agent_thought_id=agent_thought.id), PublishFrom.APPLICATION_MANAGER
                 )
-                self.queue_manager.publish(QueueAgentThoughtEvent(
-                    agent_thought_id=agent_thought.id
-                ), PublishFrom.APPLICATION_MANAGER)
 
             # update prompt tool
             for prompt_tool in prompt_messages_tools:
@@ -308,15 +292,18 @@ class FunctionCallAgentRunner(BaseAgentRunner):
 
         self.update_db_variables(self.variables_pool, self.db_variables_pool)
         # publish end event
-        self.queue_manager.publish(QueueMessageEndEvent(llm_result=LLMResult(
-            model=model_instance.model,
-            prompt_messages=prompt_messages,
-            message=AssistantPromptMessage(
-                content=final_answer
+        self.queue_manager.publish(
+            QueueMessageEndEvent(
+                llm_result=LLMResult(
+                    model=model_instance.model,
+                    prompt_messages=prompt_messages,
+                    message=AssistantPromptMessage(content=final_answer),
+                    usage=llm_usage["usage"] or LLMUsage.empty_usage(),
+                    system_fingerprint="",
+                )
             ),
-            usage=llm_usage['usage'] if llm_usage['usage'] else LLMUsage.empty_usage(),
-            system_fingerprint=''
-        )), PublishFrom.APPLICATION_MANAGER)
+            PublishFrom.APPLICATION_MANAGER,
+        )
 
     def check_tool_calls(self, llm_result_chunk: LLMResultChunk) -> bool:
         """
@@ -325,7 +312,7 @@ class FunctionCallAgentRunner(BaseAgentRunner):
         if llm_result_chunk.delta.message.tool_calls:
             return True
         return False
-    
+
     def check_blocking_tool_calls(self, llm_result: LLMResult) -> bool:
         """
         Check if there is any blocking tool call in llm result
@@ -334,7 +321,9 @@ class FunctionCallAgentRunner(BaseAgentRunner):
             return True
         return False
 
-    def extract_tool_calls(self, llm_result_chunk: LLMResultChunk) -> Union[None, list[tuple[str, str, dict[str, Any]]]]:
+    def extract_tool_calls(
+        self, llm_result_chunk: LLMResultChunk
+    ) -> Union[None, list[tuple[str, str, dict[str, Any]]]]:
         """
         Extract tool calls from llm result chunk
 
@@ -344,17 +333,19 @@ class FunctionCallAgentRunner(BaseAgentRunner):
         tool_calls = []
         for prompt_message in llm_result_chunk.delta.message.tool_calls:
             args = {}
-            if prompt_message.function.arguments != '':
+            if prompt_message.function.arguments != "":
                 args = json.loads(prompt_message.function.arguments)
 
-            tool_calls.append((
-                prompt_message.id,
-                prompt_message.function.name,
-                args,
-            ))
+            tool_calls.append(
+                (
+                    prompt_message.id,
+                    prompt_message.function.name,
+                    args,
+                )
+            )
 
         return tool_calls
-    
+
     def extract_blocking_tool_calls(self, llm_result: LLMResult) -> Union[None, list[tuple[str, str, dict[str, Any]]]]:
         """
         Extract blocking tool calls from llm result
@@ -365,18 +356,22 @@ class FunctionCallAgentRunner(BaseAgentRunner):
         tool_calls = []
         for prompt_message in llm_result.message.tool_calls:
             args = {}
-            if prompt_message.function.arguments != '':
+            if prompt_message.function.arguments != "":
                 args = json.loads(prompt_message.function.arguments)
 
-            tool_calls.append((
-                prompt_message.id,
-                prompt_message.function.name,
-                args,
-            ))
+            tool_calls.append(
+                (
+                    prompt_message.id,
+                    prompt_message.function.name,
+                    args,
+                )
+            )
 
         return tool_calls
 
-    def _init_system_message(self, prompt_template: str, prompt_messages: list[PromptMessage] = None) -> list[PromptMessage]:
+    def _init_system_message(
+        self, prompt_template: str, prompt_messages: list[PromptMessage] = None
+    ) -> list[PromptMessage]:
         """
         Initialize system message
         """
@@ -384,13 +379,13 @@ class FunctionCallAgentRunner(BaseAgentRunner):
             return [
                 SystemPromptMessage(content=prompt_template),
             ]
-        
+
         if prompt_messages and not isinstance(prompt_messages[0], SystemPromptMessage) and prompt_template:
             prompt_messages.insert(0, SystemPromptMessage(content=prompt_template))
 
         return prompt_messages
 
-    def _organize_user_query(self, query,  prompt_messages: list[PromptMessage] = None) -> list[PromptMessage]:
+    def _organize_user_query(self, query, prompt_messages: list[PromptMessage] = None) -> list[PromptMessage]:
         """
         Organize user query
         """
@@ -404,7 +399,7 @@ class FunctionCallAgentRunner(BaseAgentRunner):
             prompt_messages.append(UserPromptMessage(content=query))
 
         return prompt_messages
-    
+
     def _clear_user_prompt_image_messages(self, prompt_messages: list[PromptMessage]) -> list[PromptMessage]:
         """
         As for now, gpt supports both fc and vision at the first iteration.
@@ -415,17 +410,21 @@ class FunctionCallAgentRunner(BaseAgentRunner):
         for prompt_message in prompt_messages:
             if isinstance(prompt_message, UserPromptMessage):
                 if isinstance(prompt_message.content, list):
-                    prompt_message.content = '\n'.join([
-                        content.data if content.type == PromptMessageContentType.TEXT else 
-                        '[image]' if content.type == PromptMessageContentType.IMAGE else
-                        '[file]' 
-                        for content in prompt_message.content 
-                    ])
+                    prompt_message.content = "\n".join(
+                        [
+                            content.data
+                            if content.type == PromptMessageContentType.TEXT
+                            else "[image]"
+                            if content.type == PromptMessageContentType.IMAGE
+                            else "[file]"
+                            for content in prompt_message.content
+                        ]
+                    )
 
         return prompt_messages
 
     def _organize_prompt_messages(self):
-        prompt_template = self.app_config.prompt_template.simple_prompt_template or ''
+        prompt_template = self.app_config.prompt_template.simple_prompt_template or ""
         self.history_prompt_messages = self._init_system_message(prompt_template, self.history_prompt_messages)
         query_prompt_messages = self._organize_user_query(self.query, [])
 
@@ -433,14 +432,10 @@ class FunctionCallAgentRunner(BaseAgentRunner):
             model_config=self.model_config,
             prompt_messages=[*query_prompt_messages, *self._current_thoughts],
             history_messages=self.history_prompt_messages,
-            memory=self.memory
+            memory=self.memory,
         ).get_prompt()
 
-        prompt_messages = [
-            *self.history_prompt_messages,
-            *query_prompt_messages,
-            *self._current_thoughts
-        ]
+        prompt_messages = [*self.history_prompt_messages, *query_prompt_messages, *self._current_thoughts]
         if len(self._current_thoughts) != 0:
             # clear messages after the first iteration
             prompt_messages = self._clear_user_prompt_image_messages(prompt_messages)

api/core/agent/output_parser/cot_output_parser.py

@@ -9,8 +9,9 @@ from core.model_runtime.entities.llm_entities import LLMResultChunk
 
 class CotAgentOutputParser:
     @classmethod
-    def handle_react_stream_output(cls, llm_response: Generator[LLMResultChunk, None, None], usage_dict: dict) -> \
-        Generator[Union[str, AgentScratchpadUnit.Action], None, None]:
+    def handle_react_stream_output(
+        cls, llm_response: Generator[LLMResultChunk, None, None], usage_dict: dict
+    ) -> Generator[Union[str, AgentScratchpadUnit.Action], None, None]:
         def parse_action(json_str):
             try:
                 action = json.loads(json_str)
@@ -22,7 +23,7 @@ class CotAgentOutputParser:
                     action = action[0]
 
                 for key, value in action.items():
-                    if 'input' in key.lower():
+                    if "input" in key.lower():
                         action_input = value
                     else:
                         action_name = value
@@ -33,37 +34,37 @@ class CotAgentOutputParser:
                         action_input=action_input,
                     )
                 else:
-                    return json_str or ''
+                    return json_str or ""
             except:
-                return json_str or ''
-            
+                return json_str or ""
+
         def extra_json_from_code_block(code_block) -> Generator[Union[dict, str], None, None]:
-            code_blocks = re.findall(r'```(.*?)```', code_block, re.DOTALL)
+            code_blocks = re.findall(r"```(.*?)```", code_block, re.DOTALL)
             if not code_blocks:
                 return
             for block in code_blocks:
-                json_text = re.sub(r'^[a-zA-Z]+\n', '', block.strip(), flags=re.MULTILINE)
+                json_text = re.sub(r"^[a-zA-Z]+\n", "", block.strip(), flags=re.MULTILINE)
                 yield parse_action(json_text)
-            
-        code_block_cache = ''
+
+        code_block_cache = ""
         code_block_delimiter_count = 0
         in_code_block = False
-        json_cache = ''
+        json_cache = ""
         json_quote_count = 0
         in_json = False
         got_json = False
 
-        action_cache = ''
-        action_str = 'action:'
+        action_cache = ""
+        action_str = "action:"
         action_idx = 0
 
-        thought_cache = ''
-        thought_str = 'thought:'
+        thought_cache = ""
+        thought_str = "thought:"
         thought_idx = 0
 
         for response in llm_response:
             if response.delta.usage:
-                usage_dict['usage'] = response.delta.usage
+                usage_dict["usage"] = response.delta.usage
             response = response.delta.message.content
             if not isinstance(response, str):
                 continue
@@ -72,24 +73,24 @@ class CotAgentOutputParser:
             index = 0
             while index < len(response):
                 steps = 1
-                delta = response[index:index+steps]
-                last_character = response[index-1] if index > 0 else ''
+                delta = response[index : index + steps]
+                last_character = response[index - 1] if index > 0 else ""
 
-                if delta == '`':
+                if delta == "`":
                     code_block_cache += delta
                     code_block_delimiter_count += 1
                 else:
                     if not in_code_block:
                         if code_block_delimiter_count > 0:
                             yield code_block_cache
-                        code_block_cache = ''
+                        code_block_cache = ""
                     else:
                         code_block_cache += delta
                     code_block_delimiter_count = 0
 
                 if not in_code_block and not in_json:
                     if delta.lower() == action_str[action_idx] and action_idx == 0:
-                        if last_character not in ['\n', ' ', '']:
+                        if last_character not in {"\n", " ", ""}:
                             index += steps
                             yield delta
                             continue
@@ -97,7 +98,7 @@ class CotAgentOutputParser:
                         action_cache += delta
                         action_idx += 1
                         if action_idx == len(action_str):
-                            action_cache = ''
+                            action_cache = ""
                             action_idx = 0
                         index += steps
                         continue
@@ -105,18 +106,18 @@ class CotAgentOutputParser:
                         action_cache += delta
                         action_idx += 1
                         if action_idx == len(action_str):
-                            action_cache = ''
+                            action_cache = ""
                             action_idx = 0
                         index += steps
                         continue
                     else:
                         if action_cache:
                             yield action_cache
-                            action_cache = ''
+                            action_cache = ""
                             action_idx = 0
-                    
+
                     if delta.lower() == thought_str[thought_idx] and thought_idx == 0:
-                        if last_character not in ['\n', ' ', '']:
+                        if last_character not in {"\n", " ", ""}:
                             index += steps
                             yield delta
                             continue
@@ -124,7 +125,7 @@ class CotAgentOutputParser:
                         thought_cache += delta
                         thought_idx += 1
                         if thought_idx == len(thought_str):
-                            thought_cache = ''
+                            thought_cache = ""
                             thought_idx = 0
                         index += steps
                         continue
@@ -132,31 +133,31 @@ class CotAgentOutputParser:
                         thought_cache += delta
                         thought_idx += 1
                         if thought_idx == len(thought_str):
-                            thought_cache = ''
+                            thought_cache = ""
                             thought_idx = 0
                         index += steps
                         continue
                     else:
                         if thought_cache:
                             yield thought_cache
-                            thought_cache = ''
+                            thought_cache = ""
                             thought_idx = 0
 
                 if code_block_delimiter_count == 3:
                     if in_code_block:
                         yield from extra_json_from_code_block(code_block_cache)
-                        code_block_cache = ''
-                        
+                        code_block_cache = ""
+
                     in_code_block = not in_code_block
                     code_block_delimiter_count = 0
 
                 if not in_code_block:
                     # handle single json
-                    if delta == '{':
+                    if delta == "{":
                         json_quote_count += 1
                         in_json = True
                         json_cache += delta
-                    elif delta == '}':
+                    elif delta == "}":
                         json_cache += delta
                         if json_quote_count > 0:
                             json_quote_count -= 1
@@ -172,12 +173,12 @@ class CotAgentOutputParser:
                     if got_json:
                         got_json = False
                         yield parse_action(json_cache)
-                        json_cache = ''
+                        json_cache = ""
                         json_quote_count = 0
                         in_json = False
-                    
+
                 if not in_code_block and not in_json:
-                    yield delta.replace('`', '')
+                    yield delta.replace("`", "")
 
                 index += steps
 
@@ -186,4 +187,3 @@ class CotAgentOutputParser:
 
         if json_cache:
             yield parse_action(json_cache)
-