compatible query is None

Co-authored-by: jyong <jyong@dify.ai>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -10,7 +10,9 @@ body:
       options:
         - label: I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify/issues), including closed ones.
           required: true
-        - label: I confirm that I am using English to file this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
+        - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
+          required: true
+        - label: "Pleas do not modify this template :) and fill in all the required fields."
           required: true
 
   - type: input

.github/ISSUE_TEMPLATE/document_issue.yml

@@ -10,7 +10,9 @@ body:
       options:
         - label: I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify/issues), including closed ones.
           required: true
-        - label: I confirm that I am using English to file this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
+        - label: I confirm that I am using English to submit report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
+          required: true
+        - label: "Pleas do not modify this template :) and fill in all the required fields."
           required: true
   - type: textarea
     attributes:

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -10,26 +10,41 @@ body:
       options:
         - label: I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify/issues), including closed ones.
           required: true
-        - label: I confirm that I am using English to file this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
+        - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
+          required: true
+        - label: "Pleas do not modify this template :) and fill in all the required fields."
           required: true
   - type: textarea
     attributes:
-      label: Description of the new feature / enhancement
-      placeholder: What is the expected behavior of the proposed feature?
+      label: 1. Is this request related to a challenge you're experiencing?
+      placeholder: Please describe the specific scenario or problem you're facing as clearly as possible. For instance "I was trying to use [feature] for [specific task], and [what happened]... It was frustrating because...."
     validations:
       required: true
   - type: textarea
     attributes:
-      label: Scenario when this would be used?
-      placeholder: What is the scenario this would be used? Why is this important to your workflow as a dify user?
+      label: 2. Describe the feature you'd like to see
+      placeholder: Think about what you want to achieve and how this feature will help you. Sketches, flow diagrams, or any visual representation will be a major plus.
     validations:
       required: true
   - type: textarea
     attributes:
-      label: Supporting information
-      placeholder: "Having additional evidence, data, tweets, blog posts, research, ... anything is extremely helpful. This information provides context to the scenario that may otherwise be lost."
+      label: 3. How will this feature improve your workflow or experience?
+      placeholder: Tell us how this change will benefit your work. This helps us prioritize based on user impact.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: 4. Additional context or comments
+      placeholder: (Any other information, comments, documentations, links, or screenshots that would provide more clarity. This is the place to add anything else not covered above.)
     validations:
       required: false
+  - type: checkboxes
+    attributes:
+      label: 5. Can you help us with this feature?
+      description: Let us know! This is not a commitment, but a starting point for collaboration.
+      options:
+        - label: I am interested in contributing to this feature.
+          required: false
   - type: markdown
     attributes:
       value: Please limit one request per issue.

.github/ISSUE_TEMPLATE/help_wanted.yml

@@ -10,7 +10,9 @@ body:
       options:
         - label: I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify/issues), including closed ones.
           required: true
-        - label: I confirm that I am using English to file this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
+        - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
+          required: true
+        - label: "Pleas do not modify this template :) and fill in all the required fields."
           required: true
   - type: textarea
     attributes:

.github/ISSUE_TEMPLATE/translation_issue.yml

@@ -10,7 +10,9 @@ body:
       options:
         - label: I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify/issues), including closed ones.
           required: true
-        - label: I confirm that I am using English to file this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
+        - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
+          required: true
+        - label: "Pleas do not modify this template :) and fill in all the required fields."
           required: true
   - type: input
     attributes:

.github/pull_request_template.md

@@ -0,0 +1,32 @@
+# Description
+
+Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
+
+Fixes # (issue)
+
+## Type of Change
+
+Please delete options that are not relevant.
+
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] This change requires a documentation update, included: [Dify Document](https://github.com/langgenius/dify-docs)
+- [ ] Improvement, including but not limited to code refactoring, performance optimization, and UI/UX improvement
+- [ ] Dependency upgrade
+
+# How Has This Been Tested?
+
+Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration
+
+- [ ] TODO
+
+# Suggested Checklist:
+
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] My changes generate no new warnings
+- [ ] I ran `dev/reformat`(backend) and `cd web && npx lint-staged`(frontend) to appease the lint gods
+- [ ] `optional` I have made corresponding changes to the documentation 
+- [ ] `optional` I have added tests that prove my fix is effective or that my feature works
+- [ ] `optional` New and existing unit tests pass locally with my changes

.github/workflows/api-model-runtime-tests.yml

@@ -4,10 +4,6 @@ on:
   pull_request:
     branches:
       - main
-  push:
-    branches:
-      - deploy/dev
-      - feat/model-runtime
 
 jobs:
   test:

.github/workflows/api-tools-tests.yaml

@@ -0,0 +1,26 @@
+name: Run Tool Pytest
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+          cache-dependency-path: ./api/requirements.txt
+
+      - name: Install dependencies
+        run: pip install -r ./api/requirements.txt
+
+      - name: Run pytest
+        run: pytest ./api/tests/integration_tests/tools/test_all_provider.py

.github/workflows/api-workflow-tests.yaml

@@ -0,0 +1,31 @@
+name: Run Pytest
+
+on:
+  pull_request:
+    branches:
+      - main
+      - deploy/dev
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    env:
+      MOCK_SWITCH: true
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+          cache-dependency-path: ./api/requirements.txt
+
+      - name: Install dependencies
+        run: pip install -r ./api/requirements.txt
+
+      - name: Run pytest
+        run: pytest api/tests/integration_tests/workflow

.github/workflows/build-push.yml

@@ -1,17 +1,33 @@
-name: Build and Push API Image
+name: Build and Push API & Web
 
 on:
   push:
     branches:
-      - 'main'
-      - 'deploy/dev'
+      - "main"
+      - "deploy/dev"
+      - "feat/workflow"
   release:
-    types: [ published ]
+    types: [published]
+
+env:
+  DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+  DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+  DIFY_WEB_IMAGE_NAME: ${{ vars.DIFY_WEB_IMAGE_NAME || 'langgenius/dify-web' }}
+  DIFY_API_IMAGE_NAME: ${{ vars.DIFY_API_IMAGE_NAME || 'langgenius/dify-api' }}
 
 jobs:
   build-and-push:
     runs-on: ubuntu-latest
     if: github.event.pull_request.draft == false
+    strategy:
+      matrix:
+        include:
+          - service_name: "web"
+            image_name_env: "DIFY_WEB_IMAGE_NAME"
+            context: "web"
+          - service_name: "api"
+            image_name_env: "DIFY_API_IMAGE_NAME"
+            context: "api"
     steps:
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -22,16 +38,16 @@ jobs:
       - name: Login to Docker Hub
         uses: docker/login-action@v2
         with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
+          username: ${{ env.DOCKERHUB_USER }}
+          password: ${{ env.DOCKERHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: langgenius/dify-api
+          images: ${{ env[matrix.image_name_env] }}
           tags: |
-            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') }}
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' && startsWith(github.ref, 'refs/tags/') }}
             type=ref,event=branch
             type=sha,enable=true,priority=100,prefix=,suffix=,format=long
             type=raw,value=${{ github.ref_name }},enable=${{ startsWith(github.ref, 'refs/tags/') }}
@@ -39,22 +55,11 @@ jobs:
       - name: Build and push
         uses: docker/build-push-action@v5
         with:
-          context: "{{defaultContext}}:api"
+          context: "{{defaultContext}}:${{ matrix.context }}"
           platforms: ${{ startsWith(github.ref, 'refs/tags/') && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
-          build-args: |
-            COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
+          build-args: COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
-
-      - name: Deploy to server
-        if: github.ref == 'refs/heads/deploy/dev'
-        uses: appleboy/ssh-action@v0.1.8
-        with:
-          host: ${{ secrets.SSH_HOST }}
-          username: ${{ secrets.SSH_USER }}
-          key: ${{ secrets.SSH_PRIVATE_KEY }}
-          script: |
-            ${{ secrets.SSH_SCRIPT }}

.github/workflows/build-web-image.yml

@@ -1,60 +0,0 @@
-name: Build and Push WEB Image
-
-on:
-  push:
-    branches:
-      - 'main'
-      - 'deploy/dev'
-  release:
-    types: [ published ]
-
-jobs:
-  build-and-push:
-    runs-on: ubuntu-latest
-    if: github.event.pull_request.draft == false
-    steps:
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: langgenius/dify-web
-          tags: |
-            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') }}
-            type=ref,event=branch
-            type=sha,enable=true,priority=100,prefix=,suffix=,format=long
-            type=raw,value=${{ github.ref_name }},enable=${{ startsWith(github.ref, 'refs/tags/') }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: "{{defaultContext}}:web"
-          platforms: ${{ startsWith(github.ref, 'refs/tags/') && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
-          build-args: |
-            COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-      - name: Deploy to server
-        if: github.ref == 'refs/heads/deploy/dev'
-        uses: appleboy/ssh-action@v0.1.8
-        with:
-          host: ${{ secrets.SSH_HOST }}
-          username: ${{ secrets.SSH_USER }}
-          key: ${{ secrets.SSH_PRIVATE_KEY }}
-          script: |
-            ${{ secrets.SSH_SCRIPT }}

.github/workflows/deploy-dev.yml

@@ -0,0 +1,24 @@
+name: Deploy Dev
+
+on:
+  workflow_run:
+    workflows: ["Build and Push API & Web"]
+    branches:
+      - "deploy/dev"
+    types:
+      - completed
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    if: |
+      github.event.workflow_run.conclusion == 'success'
+    steps:
+      - name: Deploy to server
+        uses: appleboy/ssh-action@v0.1.8
+        with:
+          host: ${{ secrets.SSH_HOST }}
+          username: ${{ secrets.SSH_USER }}
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          script: |
+            ${{ vars.SSH_SCRIPT || secrets.SSH_SCRIPT }}

.github/workflows/style.yml

@@ -4,27 +4,50 @@ on:
   pull_request:
     branches:
       - main
-  push:
-    branches:
-      - deploy/dev
 
 concurrency:
   group: dep-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
 jobs:
-  test:
-    name: ESLint and SuperLinter
+  python-style:
+    name: Python Style
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Python dependencies
+        run: pip install ruff
+
+      - name: Ruff check
+        run: ruff check ./api
+
+      - name: Lint hints
+        if: failure()
+        run: echo "Please run 'dev/reformat' to fix the fixable linting errors."
+
+  test:
+    name: ESLint and SuperLinter
+    runs-on: ubuntu-latest
+    needs: python-style
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
       - name: Setup NodeJS
         uses: actions/setup-node@v4
         with:
-          node-version: 18
+          node-version: 20
           cache: yarn
           cache-dependency-path: ./web/package.json
 
@@ -39,11 +62,10 @@ jobs:
           yarn run lint
 
       - name: Super-linter
-        uses: super-linter/super-linter/slim@v5
+        uses: super-linter/super-linter/slim@v6
         env:
           BASH_SEVERITY: warning
           DEFAULT_BRANCH: main
-          ERROR_ON_MISSING_EXEC_BIT: true
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           IGNORE_GENERATED_FILES: true
           IGNORE_GITIGNORED_FILES: true

.github/workflows/tool-test-sdks.yaml

@@ -0,0 +1,34 @@
+name: Run Unit Test For SDKs
+
+on:
+  pull_request:
+    branches:
+      - main
+jobs:
+  build:
+    name: unit test for Node.js SDK
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [16, 18, 20]
+
+    defaults:
+      run:
+        working-directory: sdks/nodejs-client
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: ''
+          cache-dependency-path: 'yarn.lock'
+
+      - name: Install Dependencies
+        run: yarn install
+
+      - name: Test
+        run: yarn test

.gitignore

@@ -145,10 +145,14 @@ docker/volumes/db/data/*
 docker/volumes/redis/data/*
 docker/volumes/weaviate/*
 docker/volumes/qdrant/*
+docker/volumes/etcd/*
+docker/volumes/minio/*
+docker/volumes/milvus/*
 
 sdks/python-client/build
 sdks/python-client/dist
 sdks/python-client/dify_client.egg-info
 
 .vscode/*
-!.vscode/launch.json
+!.vscode/launch.json
+pyrightconfig.json

CONTRIBUTING.md

@@ -1,66 +1,158 @@
-# Contributing
+So you're looking to contribute to Dify - that's awesome, we can't wait to see what you do. As a startup with limited headcount and funding, we have grand ambitions to design the most intuitive workflow for building and managing LLM applications. Any help from the community counts, truly.
 
-Thanks for your interest in [Dify](https://dify.ai) and for wanting to contribute! Before you begin, read the
-[code of conduct](https://github.com/langgenius/.github/blob/main/CODE_OF_CONDUCT.md) and check out the
-[existing issues](https://github.com/langgenius/langgenius-gateway/issues).
-This document describes how to set up your development environment to build and test [Dify](https://dify.ai).
+We need to be nimble and ship fast given where we are, but we also want to make sure that contributors like you get as smooth an experience at contributing as possible. We've assembled this contribution guide for that purpose, aiming at getting you familiarized with the codebase & how we work with contributors, so you could quickly jump to the fun part. 
 
-### Install dependencies
+This guide, like Dify itself, is a constant work in progress. We highly appreciate your understanding if at times it lags behind the actual project, and welcome any feedback for us to improve.
 
-You need to install and configure the following dependencies on your machine to build [Dify](https://dify.ai):
+In terms of licensing, please take a minute to read our short [License and Contributor Agreement](./license). The community also adheres to the [code of conduct](https://github.com/langgenius/.github/blob/main/CODE_OF_CONDUCT.md).
+
+## Before you jump in
+
+[Find](https://github.com/langgenius/dify/issues?q=is:issue+is:closed) an existing issue, or [open](https://github.com/langgenius/dify/issues/new/choose) a new one. We categorize issues into 2 types:
+
+### Feature requests:
+
+* If you're opening a new feature request, we'd like you to explain what the proposed feature achieves, and include as much context as possible. [@perzeusss](https://github.com/perzeuss) has made a solid [Feature Request Copilot](https://udify.app/chat/MK2kVSnw1gakVwMX) that helps you draft out your needs. Feel free to give it a try.
+
+* If you want to pick one up from the existing issues, simply drop a comment below it saying so.
+
+  
+
+  A team member working in the related direction will be looped in. If all looks good, they will give the go-ahead for you to start coding. We ask that you hold off working on the feature until then, so none of your work goes to waste should we propose changes.
+
+  Depending on whichever area the proposed feature falls under, you might talk to different team members. Here's rundown of the areas each our team members are working on at the moment:
+
+  | Member                                                       | Scope                                                |
+  | ------------------------------------------------------------ | ---------------------------------------------------- |
+  | [@yeuoly](https://github.com/Yeuoly)                         | Architecting Agents                                  |
+  | [@jyong](https://github.com/JohnJyong)                       | RAG pipeline design                                  |
+  | [@GarfieldDai](https://github.com/GarfieldDai)               | Building workflow orchestrations                     |
+  | [@iamjoel](https://github.com/iamjoel) & [@zxhlyh](https://github.com/zxhlyh) | Making our frontend a breeze to use                  |
+  | [@guchenhe](https://github.com/guchenhe) & [@crazywoola](https://github.com/crazywoola) | Developer experience, points of contact for anything |
+  | [@takatost](https://github.com/takatost)                     | Overall product direction and architecture           |
+
+  How we prioritize:
+
+  | Feature Type                                                 | Priority        |
+  | ------------------------------------------------------------ | --------------- |
+  | High-Priority Features as being labeled by a team member     | High Priority   |
+  | Popular feature requests from our [community feedback board](https://feedback.dify.ai/) | Medium Priority |
+  | Non-core features and minor enhancements                     | Low Priority    |
+  | Valuable but not immediate                                   | Future-Feature  |
+
+### Anything else (e.g. bug report, performance optimization, typo correction):
+
+* Start coding right away.
+
+  How we prioritize:
+
+  | Issue Type                                                   | Priority        |
+  | ------------------------------------------------------------ | --------------- |
+  | Bugs in core functions (cannot login, applications not working, security loopholes) | Critical        |
+  | Non-critical bugs, performance boosts                        | Medium Priority |
+  | Minor fixes (typos, confusing but working UI)                | Low Priority    |
+
+
+## Installing
+
+Here are the steps to set up Dify for development:
+
+### 1. Fork this repository
+
+### 2. Clone the repo
+
+ Clone the forked repository from your terminal:
+
+```
+git clone git@github.com:<github_username>/dify.git
+```
+
+### 3. Verify dependencies
+
+Dify requires the following dependencies to build, make sure they're installed on your system:
 
-- [Git](http://git-scm.com/)
 - [Docker](https://www.docker.com/)
 - [Docker Compose](https://docs.docker.com/compose/install/)
 - [Node.js v18.x (LTS)](http://nodejs.org)
 - [npm](https://www.npmjs.com/) version 8.x.x or [Yarn](https://yarnpkg.com/)
 - [Python](https://www.python.org/) version 3.10.x
 
-## Local development
+### 4. Installations
 
-To set up a working development environment, just fork the project git repository and install the backend and frontend dependencies using the proper package manager and create run the docker-compose stack.
+Dify is composed of a backend and a frontend. Navigate to the backend directory by `cd api/`, then follow the [Backend README](api/README.md) to install it. In a separate terminal, navigate to the frontend directory by `cd web/`, then follow the [Frontend README](web/README.md) to install.
 
-### Fork the repository
+Check the [installation FAQ](https://docs.dify.ai/getting-started/faq/install-faq) for a list of common issues and steps to troubleshoot.
 
-you need to fork the [repository](https://github.com/langgenius/dify).
+### 5. Visit dify in your browser
 
-### Clone the repo
+To validate your set up, head over to [http://localhost:3000](http://localhost:3000) (the default, or your self-configured URL and port) in your browser. You should now see Dify up and running. 
 
-Clone your GitHub forked repository:
+## Developing
+
+If you are adding a model provider, [this guide](https://github.com/langgenius/dify/blob/main/api/core/model_runtime/README.md) is for you.
+
+If you are adding a tool provider to Agent or Workflow, [this guide](./api/core/tools/README.md) is for you.
+
+To help you quickly navigate where your contribution fits, a brief, annotated outline of Dify's backend & frontend is as follows:
+
+### Backend
+
+Dify’s backend is written in Python using [Flask](https://flask.palletsprojects.com/en/3.0.x/). It uses [SQLAlchemy](https://www.sqlalchemy.org/) for ORM and [Celery](https://docs.celeryq.dev/en/stable/getting-started/introduction.html) for task queueing. Authorization logic goes via Flask-login. 
 
 ```
-git clone git@github.com:<github_username>/dify.git
+[api/]
+├── constants             // Constant settings used throughout code base.
+├── controllers           // API route definitions and request handling logic.           
+├── core                  // Core application orchestration, model integrations, and tools.
+├── docker                // Docker & containerization related configurations.
+├── events                // Event handling and processing
+├── extensions            // Extensions with 3rd party frameworks/platforms.
+├── fields                // field definitions for serialization/marshalling.
+├── libs                  // Reusable libraries and helpers.
+├── migrations            // Scripts for database migration.
+├── models                // Database models & schema definitions.
+├── services              // Specifies business logic.
+├── storage               // Private key storage.      
+├── tasks                 // Handling of async tasks and background jobs.
+└── tests
 ```
 
-### Install backend
+### Frontend
 
-To learn how to install the backend application, please refer to the [Backend README](api/README.md).
+The website is bootstrapped on [Next.js](https://nextjs.org/) boilerplate in Typescript and uses [Tailwind CSS](https://tailwindcss.com/) for styling. [React-i18next](https://react.i18next.com/) is used for internationalization.
 
-### Install frontend
+```
+[web/]
+├── app                   // layouts, pages, and components
+│   ├── (commonLayout)    // common layout used throughout the app
+│   ├── (shareLayout)     // layouts specifically shared across token-specific sessions 
+│   ├── activate          // activate page
+│   ├── components        // shared by pages and layouts
+│   ├── install           // install page
+│   ├── signin            // signin page
+│   └── styles            // globally shared styles
+├── assets                // Static assets
+├── bin                   // scripts ran at build step
+├── config                // adjustable settings and options 
+├── context               // shared contexts used by different portions of the app
+├── dictionaries          // Language-specific translate files 
+├── docker                // container configurations
+├── hooks                 // Reusable hooks
+├── i18n                  // Internationalization configuration
+├── models                // describes data models & shapes of API responses
+├── public                // meta assets like favicon
+├── service               // specifies shapes of API actions
+├── test                  
+├── types                 // descriptions of function params and return values
+└── utils                 // Shared utility functions
+```
 
-To learn how to install the frontend application, please refer to the [Frontend README](web/README.md).
+## Submitting your PR
 
-### Visit dify in your browser
+At last, time to open a pull request (PR) to our repo. For major features, we first merge them into the `deploy/dev` branch for testing, before they go into the `main` branch. If you run into issues like merge conflicts or don't know how to open a pull request, check out [GitHub's pull request tutorial](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests). 
 
-Finally, you can now visit [http://localhost:3000](http://localhost:3000) to view the [Dify](https://dify.ai) in local environment.
+And that's it! Once your PR is merged, you will be featured as a contributor in our [README](https://github.com/langgenius/dify/blob/main/README.md).
 
+## Getting Help
 
-## Create a pull request
-
-After making your changes, open a pull request (PR). Once you submit your pull request, others from the Dify team/community will review it with you.
-
-Did you have an issue, like a merge conflict, or don't know how to open a pull request? Check out [GitHub's pull request tutorial](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests) on how to resolve merge conflicts and other issues. Once your PR has been merged, you will be proudly listed as a contributor in the [contributor chart](https://github.com/langgenius/langgenius-gateway/graphs/contributors).
-
-## Community channels
-
-Stuck somewhere? Have any questions? Join the [Discord Community Server](https://discord.gg/j3XRWSPBf7). We are here to help!
-
-
-### Provider Integrations
-If you see a model provider not yet supported by Dify that you'd like to use, follow these [steps](api/core/model_runtime/README.md) to submit a PR.
-
-
-### i18n (Internationalization) Support
-
-We are looking for contributors to help with translations in other languages. If you are interested in helping, please join the [Discord Community Server](https://discord.gg/AhzKf7dNgk) and let us know.  
-Also check out the [Frontend i18n README](web/i18n/README_EN.md) for more information.
+If you ever get stuck or got a burning question while contributing, simply shoot your queries our way via the related GitHub issue, or hop onto our [Discord](https://discord.gg/8Tpq4AcN9c) for a quick chat. 

CONTRIBUTING_CN.md

@@ -1,57 +1,155 @@
-# 贡献
+所以你想为 Dify 做贡献 - 这太棒了，我们迫不及待地想看到你的贡献。作为一家人员和资金有限的初创公司，我们有着雄心勃勃的目标，希望设计出最直观的工作流程来构建和管理 LLM 应用程序。社区的任何帮助都是宝贵的。
 
-感谢您对 [Dify](https://dify.ai) 的兴趣，并希望您能够做出贡献！在开始之前，请先阅读[行为准则](https://github.com/langgenius/.github/blob/main/CODE_OF_CONDUCT.md)并查看[现有问题](https://github.com/langgenius/dify/issues)。
-本文档介绍了如何设置开发环境以构建和测试 [Dify](https://dify.ai)。
+考虑到我们的现状，我们需要灵活快速地交付，但我们也希望确保像你这样的贡献者在贡献过程中获得尽可能顺畅的体验。我们为此编写了这份贡献指南，旨在让你熟悉代码库和我们与贡献者的合作方式，以便你能快速进入有趣的部分。
 
-### 安装依赖项
+这份指南，就像 Dify 本身一样，是一个不断改进的工作。如果有时它落后于实际项目，我们非常感谢你的理解，并欢迎任何反馈以供我们改进。
 
-您需要在计算机上安装和配置以下依赖项才能构建 [Dify](https://dify.ai)：
+在许可方面，请花一分钟阅读我们简短的[许可证和贡献者协议](./license)。社区还遵守[行为准则](https://github.com/langgenius/.github/blob/main/CODE_OF_CONDUCT.md)。
 
-- [Git](http://git-scm.com/)
-- [Docker](https://www.docker.com/)
-- [Docker Compose](https://docs.docker.com/compose/install/)
-- [Node.js v18.x (LTS)](http://nodejs.org)
-- [npm](https://www.npmjs.com/) 版本 8.x.x 或 [Yarn](https://yarnpkg.com/)
-- [Python](https://www.python.org/) 版本 3.10.x
+## 在开始之前
 
-## 本地开发
+[查找](https://github.com/langgenius/dify/issues?q=is:issue+is:closed)现有问题，或[创建](https://github.com/langgenius/dify/issues/new/choose)一个新问题。我们将问题分为两类：
 
-要设置一个可工作的开发环境，只需 fork 项目的 git 存储库，并使用适当的软件包管理器安装后端和前端依赖项，然后创建并运行 docker-compose。
+### 功能请求：
 
-### Fork存储库
+* 如果您要提出新的功能请求，请解释所提议的功能的目标，并尽可能提供详细的上下文。[@perzeusss](https://github.com/perzeuss)制作了一个很好的[功能请求助手](https://udify.app/chat/MK2kVSnw1gakVwMX)，可以帮助您起草需求。随时尝试一下。
 
-您需要 fork [Git 仓库](https://github.com/langgenius/dify)。
+* 如果您想从现有问题中选择一个，请在其下方留下评论表示您的意愿。
 
-### 克隆存储库
+相关方向的团队成员将参与其中。如果一切顺利，他们将批准您开始编码。在此之前，请不要开始工作，以免我们提出更改导致您的工作付诸东流。
 
-克隆您在 GitHub 上 fork 的仓库：
+根据所提议的功能所属的领域不同，您可能需要与不同的团队成员交流。以下是我们团队成员目前正在从事的各个领域的概述：
+
+  | Member                                                       | Scope                                                |
+  | ------------------------------------------------------------ | ---------------------------------------------------- |
+  | [@yeuoly](https://github.com/Yeuoly)                         | Architecting Agents                                  |
+  | [@jyong](https://github.com/JohnJyong)                       | RAG pipeline design                                  |
+  | [@GarfieldDai](https://github.com/GarfieldDai)               | Building workflow orchestrations                     |
+  | [@iamjoel](https://github.com/iamjoel) & [@zxhlyh](https://github.com/zxhlyh) | Making our frontend a breeze to use                  |
+  | [@guchenhe](https://github.com/guchenhe) & [@crazywoola](https://github.com/crazywoola) | Developer experience, points of contact for anything |
+  | [@takatost](https://github.com/takatost)                     | Overall product direction and architecture           |
+
+  How we prioritize:
+
+  | Feature Type                                                 | Priority        |
+  | ------------------------------------------------------------ | --------------- |
+  | High-Priority Features as being labeled by a team member     | High Priority   |
+  | Popular feature requests from our [community feedback board](https://feedback.dify.ai/) | Medium Priority |
+  | Non-core features and minor enhancements                     | Low Priority    |
+  | Valuable but not immediate                                   | Future-Feature  |
+
+### 其他任何事情（例如bug报告、性能优化、拼写错误更正）：
+* 立即开始编码。
+
+  How we prioritize:
+
+  | Issue Type                                                   | Priority        |
+  | ------------------------------------------------------------ | --------------- |
+  | Bugs in core functions (cannot login, applications not working, security loopholes) | Critical        |
+  | Non-critical bugs, performance boosts                        | Medium Priority |
+  | Minor fixes (typos, confusing but working UI)                | Low Priority    |
+
+
+## 安装
+
+以下是设置Dify进行开发的步骤：
+
+### 1. Fork该仓库
+
+### 2. 克隆仓库
+
+从终端克隆fork的仓库：
 
 ```
 git clone git@github.com:<github_username>/dify.git
 ```
 
-### 安装后端
+### 3. 验证依赖项
 
-要了解如何安装后端应用程序，请参阅[后端 README](api/README.md)。
+Dify 依赖以下工具和库：
 
-### 安装前端
+- [Docker](https://www.docker.com/)
+- [Docker Compose](https://docs.docker.com/compose/install/)
+- [Node.js v18.x (LTS)](http://nodejs.org)
+- [npm](https://www.npmjs.com/) version 8.x.x or [Yarn](https://yarnpkg.com/)
+- [Python](https://www.python.org/) version 3.10.x
 
-要了解如何安装前端应用程序，请参阅[前端 README](web/README.md)。
+### 4. 安装
 
-### 在浏览器中访问 Dify
+Dify由后端和前端组成。通过`cd api/`导航到后端目录，然后按照[后端README](api/README.md)进行安装。在另一个终端中，通过`cd web/`导航到前端目录，然后按照[前端README](web/README.md)进行安装。
 
-最后，您现在可以访问 [http://localhost:3000](http://localhost:3000) 在本地环境中查看 [Dify](https://dify.ai)。
+查看[安装常见问题解答](https://docs.dify.ai/getting-started/faq/install-faq)以获取常见问题列表和故障排除步骤。
 
-## 创建拉取请求
+### 5. 在浏览器中访问Dify
 
-在进行更改后，打开一个拉取请求（PR）。提交拉取请求后，Dify 团队/社区的其他人将与您一起审查它。
+为了验证您的设置，打开浏览器并访问[http://localhost:3000](http://localhost:3000)（默认或您自定义的URL和端口）。现在您应该看到Dify正在运行。
 
-如果遇到问题，比如合并冲突或不知道如何打开拉取请求，请查看 GitHub 的[拉取请求教程](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests)，了解如何解决合并冲突和其他问题。一旦您的 PR 被合并，您将自豪地被列为[贡献者表](https://github.com/langgenius/dify/graphs/contributors)中的一员。
+## 开发
 
-## 社区渠道
+如果您要添加模型提供程序，请参考[此指南](https://github.com/langgenius/dify/blob/main/api/core/model_runtime/README.md)。
 
-遇到困难了吗？有任何问题吗? 加入 [Discord Community Server](https://discord.gg/AhzKf7dNgk)，我们将为您提供帮助。
+如果您要向Agent或Workflow添加工具提供程序，请参考[此指南](./api/core/tools/README.md)。
 
-### 多语言支持
+为了帮助您快速了解您的贡献在哪个部分，以下是Dify后端和前端的简要注释大纲：
 
-需要参与贡献翻译内容，请参阅[前端多语言翻译 README](web/i18n/README_CN.md)。
+### 后端
+
+Dify的后端使用Python编写，使用[Flask](https://flask.palletsprojects.com/en/3.0.x/)框架。它使用[SQLAlchemy](https://www.sqlalchemy.org/)作为ORM，使用[Celery](https://docs.celeryq.dev/en/stable/getting-started/introduction.html)作为任务队列。授权逻辑通过Flask-login进行处理。
+
+```
+[api/]
+├── constants             // Constant settings used throughout code base.
+├── controllers           // API route definitions and request handling logic.           
+├── core                  // Core application orchestration, model integrations, and tools.
+├── docker                // Docker & containerization related configurations.
+├── events                // Event handling and processing
+├── extensions            // Extensions with 3rd party frameworks/platforms.
+├── fields                // field definitions for serialization/marshalling.
+├── libs                  // Reusable libraries and helpers.
+├── migrations            // Scripts for database migration.
+├── models                // Database models & schema definitions.
+├── services              // Specifies business logic.
+├── storage               // Private key storage.      
+├── tasks                 // Handling of async tasks and background jobs.
+└── tests
+```
+
+### 前端
+
+该网站使用基于Typescript的[Next.js](https://nextjs.org/)模板进行引导，并使用[Tailwind CSS](https://tailwindcss.com/)进行样式设计。[React-i18next](https://react.i18next.com/)用于国际化。
+
+```
+[web/]
+├── app                   // layouts, pages, and components
+│   ├── (commonLayout)    // common layout used throughout the app
+│   ├── (shareLayout)     // layouts specifically shared across token-specific sessions 
+│   ├── activate          // activate page
+│   ├── components        // shared by pages and layouts
+│   ├── install           // install page
+│   ├── signin            // signin page
+│   └── styles            // globally shared styles
+├── assets                // Static assets
+├── bin                   // scripts ran at build step
+├── config                // adjustable settings and options 
+├── context               // shared contexts used by different portions of the app
+├── dictionaries          // Language-specific translate files 
+├── docker                // container configurations
+├── hooks                 // Reusable hooks
+├── i18n                  // Internationalization configuration
+├── models                // describes data models & shapes of API responses
+├── public                // meta assets like favicon
+├── service               // specifies shapes of API actions
+├── test                  
+├── types                 // descriptions of function params and return values
+└── utils                 // Shared utility functions
+```
+
+## 提交你的 PR
+
+最后，是时候向我们的仓库提交一个拉取请求（PR）了。对于重要的功能，我们首先将它们合并到 `deploy/dev` 分支进行测试，然后再合并到 `main` 分支。如果你遇到合并冲突或者不知道如何提交拉取请求的问题，请查看 [GitHub 的拉取请求教程](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests)。
+
+就是这样！一旦你的 PR 被合并，你将成为我们 [README](https://github.com/langgenius/dify/blob/main/README.md) 中的贡献者。
+
+## 获取帮助
+
+如果你在贡献过程中遇到困难或者有任何问题，可以通过相关的 GitHub 问题提出你的疑问，或者加入我们的 [Discord](https://discord.gg/8Tpq4AcN9c) 进行快速交流。

CONTRIBUTING_JA.md

@@ -1,55 +0,0 @@
-# コントリビュート
-
-[Dify](https://dify.ai) に興味を持ち、貢献したいと思うようになったことに感謝します！始める前に、
-[行動規範](https://github.com/langgenius/.github/blob/main/CODE_OF_CONDUCT.md)を読み、
-[既存の問題](https://github.com/langgenius/langgenius-gateway/issues)をチェックしてください。
-本ドキュメントは、[Dify](https://dify.ai) をビルドしてテストするための開発環境の構築方法を説明するものです。
-
-### 依存関係のインストール
-
-[Dify](https://dify.ai)をビルドするには、お使いのマシンに以下の依存関係をインストールし、設定する必要があります:
-
-- [Git](http://git-scm.com/)
-- [Docker](https://www.docker.com/)
-- [Docker Compose](https://docs.docker.com/compose/install/)
-- [Node.js v18.x (LTS)](http://nodejs.org)
-- [npm](https://www.npmjs.com/) バージョン 8.x.x もしくは [Yarn](https://yarnpkg.com/)
-- [Python](https://www.python.org/) バージョン 3.10.x
-
-## ローカル開発
-
-開発環境を構築するには、プロジェクトの git リポジトリをフォークし、適切なパッケージマネージャを使用してバックエンドとフロントエンドの依存関係をインストールし、docker-compose スタックを実行するように作成します。
-
-### リポジトリのフォーク
-
-[リポジトリ](https://github.com/langgenius/dify) をフォークする必要があります。
-
-### リポジトリのクローン
-
-GitHub でフォークしたリポジトリのクローンを作成する:
-
-```
-git clone git@github.com:<github_username>/dify.git
-```
-
-### バックエンドのインストール
-
-バックエンドアプリケーションのインストール方法については、[Backend README](api/README.md) を参照してください。
-
-### フロントエンドのインストール
-
-フロントエンドアプリケーションのインストール方法については、[Frontend README](web/README.md) を参照してください。
-
-### ブラウザで dify にアクセス
-
-[Dify](https://dify.ai) をローカル環境で見ることができるようになりました [http://localhost:3000](http://localhost:3000)。
-
-## プルリクエストの作成
-
-変更後、プルリクエスト (PR) をオープンしてください。プルリクエストを提出すると、Dify チーム/コミュニティの他の人があなたと一緒にそれをレビューします。
-
-マージコンフリクトなどの問題が発生したり、プルリクエストの開き方がわからなくなったりしませんでしたか？ [GitHub's pull request tutorial](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests) で、マージコンフリクトやその他の問題を解決する方法をチェックしてみてください。あなたの PR がマージされると、[コントリビュータチャート](https://github.com/langgenius/langgenius-gateway/graphs/contributors)にコントリビュータとして誇らしげに掲載されます。
-
-## コミュニティチャンネル
-
-お困りですか？何か質問がありますか？ [Discord Community サーバ](https://discord.gg/j3XRWSPBf7) に参加してください。私たちがお手伝いします！

LICENSE

@@ -1,24 +1,26 @@
-# Dify Open Source License
+# Open Source License
 
-The Dify project is licensed under the Apache License 2.0, with the following additional conditions:
+Dify is licensed under the Apache License 2.0, with the following additional conditions:
 
-1. Dify is permitted to be used for commercialization, such as using Dify as a "backend-as-a-service" for your other applications, or delivering it to enterprises as an application development platform. However, when the following conditions are met, you must contact the producer to obtain a commercial license:
+1. Dify may be utilized commercially, including as a backend service for other applications or as an application development platform for enterprises. Should the conditions below be met, a commercial license must be obtained from the producer:
 
-a. Multi-tenant SaaS service: Unless explicitly authorized by Dify in writing, you may not use the Dify.AI source code to operate a multi-tenant SaaS service that is similar to the Dify.AI service edition.
-b. LOGO and copyright information: In the process of using Dify, you may not remove or modify the LOGO or copyright information in the Dify console.
+a. Multi-tenant SaaS service: Unless explicitly authorized by Dify in writing, you may not use the Dify source code to operate a multi-tenant environment. 
+    - Tenant Definition: Within the context of Dify, one tenant corresponds to one workspace. The workspace provides a separated area for each tenant's data and configurations.
+
+b. LOGO and copyright information: In the process of using Dify's frontend components, you may not remove or modify the LOGO or copyright information in the Dify console or applications. This restriction is inapplicable to uses of Dify that do not involve its frontend components.
 
 Please contact business@dify.ai by email to inquire about licensing matters.
 
-2. As a contributor, you should agree that your contributed code:
+2. As a contributor, you should agree that:
 
-a. The producer can adjust the open-source agreement to be more strict or relaxed.
-b. Can be used for commercial purposes, such as Dify's cloud business.
+a. The producer can adjust the open-source agreement to be more strict or relaxed as deemed necessary.
+b. Your contributed code may be used for commercial purposes, including but not limited to its cloud business operations.
 
-Apart from this, all other rights and restrictions follow the Apache License 2.0. If you need more detailed information, you can refer to the full version of Apache License 2.0.
+Apart from the specific conditions mentioned above, all other rights and restrictions follow the Apache License 2.0. Detailed information about the Apache License 2.0 can be found at http://www.apache.org/licenses/LICENSE-2.0.
 
 The interactive design of this product is protected by appearance patent.
 
-© 2023 LangGenius, Inc.
+© 2024 LangGenius, Inc.
 
 
 ----------

Makefile

@@ -0,0 +1,43 @@
+# Variables
+DOCKER_REGISTRY=langgenius
+WEB_IMAGE=$(DOCKER_REGISTRY)/dify-web
+API_IMAGE=$(DOCKER_REGISTRY)/dify-api
+VERSION=latest
+
+# Build Docker images
+build-web:
+	@echo "Building web Docker image: $(WEB_IMAGE):$(VERSION)..."
+	docker build -t $(WEB_IMAGE):$(VERSION) ./web
+	@echo "Web Docker image built successfully: $(WEB_IMAGE):$(VERSION)"
+
+build-api:
+	@echo "Building API Docker image: $(API_IMAGE):$(VERSION)..."
+	docker build -t $(API_IMAGE):$(VERSION) ./api
+	@echo "API Docker image built successfully: $(API_IMAGE):$(VERSION)"
+
+# Push Docker images
+push-web:
+	@echo "Pushing web Docker image: $(WEB_IMAGE):$(VERSION)..."
+	docker push $(WEB_IMAGE):$(VERSION)
+	@echo "Web Docker image pushed successfully: $(WEB_IMAGE):$(VERSION)"
+
+push-api:
+	@echo "Pushing API Docker image: $(API_IMAGE):$(VERSION)..."
+	docker push $(API_IMAGE):$(VERSION)
+	@echo "API Docker image pushed successfully: $(API_IMAGE):$(VERSION)"
+
+# Build all images
+build-all: build-web build-api
+
+# Push all images
+push-all: push-web push-api
+
+build-push-api: build-api push-api
+build-push-web: build-web push-web
+
+# Build and push all images
+build-push-all: build-all push-all
+	@echo "All Docker images have been built and pushed."
+
+# Phony targets
+.PHONY: build-web build-api push-web push-api build-all push-all build-push-all

README.md

@@ -21,28 +21,38 @@
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web"></a>
 </p>
 
+<p align="center">
+   <a href="https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6" target="_blank">
+   📌 Check out Dify Premium on AWS and deploy it to your own AWS VPC with one-click.
+  </a>
+</p>
 
-**Dify** is an LLM application development platform that has helped built over **100,000** applications. It integrates BaaS and LLMOps, covering the essential tech stack for building generative AI-native applications, including a built-in RAG engine. Dify allows you to **deploy your own version of Assistants API and GPTs, based on any LLMs.**
+**Dify** is an open-source LLM app development platform. Dify's intuitive interface combines a RAG pipeline, AI workflow orchestration, agent capabilities, model management, observability features and more, letting you quickly go from prototype to production.
 
 ![](./images/demo.png)
 
-## Use Cloud Services
 
-[Dify.AI Cloud](https://dify.ai) provides all the capabilities of the open-source version, and includes 200 free requests to OpenAI GPT-3.5.
 
-## Why Dify
+## Using our Cloud Services
 
-Dify is model-agnostic and boasts a comprehensive tech stack compared to hardcoded development libraries like LangChain. Unlike OpenAI's Assistants API, Dify allows for full local deployment of services.
+You can try out [Dify.AI Cloud](https://dify.ai) now. It provides all the capabilities of the self-deployed version, and includes 200 free requests to OpenAI GPT-3.5.
+
+### Looking to purchase via AWS?
+Check out [Dify Premium on AWS](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6) and deploy it to your own AWS VPC with one-click. 
+
+## Dify vs. LangChain vs. Assistants API
 
 | Feature | Dify.AI | Assistants API | LangChain |
 |---------|---------|----------------|-----------|
 | **Programming Approach** | API-oriented | API-oriented | Python Code-oriented |
-| **Ecosystem Strategy** | Open Source | Closed and Commercial | Open Source |
+| **Ecosystem Strategy** | Open Source | Close Source | Open Source |
 | **RAG Engine** | Supported | Supported | Not Supported |
 | **Prompt IDE** | Included | Included | None |
-| **Supported LLMs** | Rich Variety | Only GPT | Rich Variety |
+| **Supported LLMs** | Rich Variety | OpenAI-only | Rich Variety |
 | **Local Deployment** | Supported | Not Supported | Not Applicable |
 
+
+
 ## Features
 
 ![](./images/models.png)
@@ -53,13 +63,14 @@ Dify is model-agnostic and boasts a comprehensive tech stack compared to hardcod
 
 **3. RAG Engine**: Includes various RAG capabilities based on full-text indexing or vector database embeddings, allowing direct upload of PDFs, TXTs, and other text formats.
 
-**4. Agents**: A Function Calling based Agent framework that allows users to configure what they see is what they get. Dify includes basic plugin capabilities like Google Search.
+**4. AI Agent**: Based on Function Calling and ReAct, the Agent inference framework allows users to customize tools, what you see is what you get. Dify provides more than a dozen built-in tool calling capabilities, such as Google Search, DELL·E, Stable Diffusion, WolframAlpha, etc.
+
 
 **5. Continuous Operations**: Monitor and analyze application logs and performance, continuously improving Prompts, datasets, or models using production data.
 
 ## Before You Start
 
-**Star us, and you'll get instant notifications for all new releases on GitHub!**
+**Star us on GitHub, and be instantly notified for new releases!**
 
 ![star-us](https://github.com/langgenius/dify/assets/100913391/95f37259-7370-4456-a9f0-0bc01ef8642f)
 
@@ -89,10 +100,12 @@ docker compose up -d
 
 After running, you can access the Dify dashboard in your browser at [http://localhost/install](http://localhost/install) and start the initialization installation process.
 
-### Helm Chart
+#### Deploy with Helm Chart
 
-Big thanks to @BorisPolonsky for providing us with a [Helm Chart](https://helm.sh/) version, which allows Dify to be deployed on Kubernetes.
-You can go to https://github.com/BorisPolonsky/dify-helm for deployment information.
+[Helm Chart](https://helm.sh/) version, which allows Dify to be deployed on Kubernetes.
+
+- [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
+- [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 
 ### Configuration
 
@@ -103,17 +116,43 @@ If you need to customize the configuration, please refer to the comments in our
 
 [![Star History Chart](https://api.star-history.com/svg?repos=langgenius/dify&type=Date)](https://star-history.com/#langgenius/dify&Date)
 
+## Contributing
+
+For those who'd like to contribute code, see our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). 
+
+At the same time, please consider supporting Dify by sharing it on social media and at events and conferences.
+
+### Projects made by community
+
+- [Chatbot Chrome Extension by @charli117](https://github.com/langgenius/chatbot-chrome-extension)
+
+### Contributors
+
+<a href="https://github.com/langgenius/dify/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=langgenius/dify" />
+</a>
+
+### Translations
+
+We are looking for contributors to help with translating Dify to languages other than Mandarin or English. If you are interested in helping, please see the [i18n README](https://github.com/langgenius/dify/blob/main/web/i18n/README.md) for more information, and leave us a comment in the `global-users` channel of our [Discord Community Server](https://discord.gg/8Tpq4AcN9c).
 
 ## Community & Support
 
-We welcome you to contribute to Dify to help make Dify better in various ways, submitting code, issues, new ideas, or sharing the interesting and useful AI applications you have created based on Dify. At the same time, we also welcome you to share Dify at different events, conferences, and social media.
+* [Canny](https://feedback.dify.ai/). Best for: sharing feedback and checking out our feature roadmap.
+* [GitHub Issues](https://github.com/langgenius/dify/issues). Best for: bugs you encounter using Dify.AI, and feature proposals. See our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
+* [Email Support](mailto:hello@dify.ai?subject=[GitHub]Questions%20About%20Dify). Best for: questions you have about using Dify.AI.
+* [Discord](https://discord.gg/FngNHpbcY7). Best for: sharing your applications and hanging out with the community.
+* [Twitter](https://twitter.com/dify_ai). Best for: sharing your applications and hanging out with the community.
+* [Business Contact](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry). Best for: business inquiries of licensing Dify.AI for commercial use.
 
-- [Roadmap and Feedback](https://feedback.dify.ai/). Best for: sharing feedback and checking out our feature roadmap.
-- [GitHub Issues](https://github.com/langgenius/dify/issues). Best for: bugs and errors you encounter using Dify.AI, see the [Contribution Guide](CONTRIBUTING.md).
-- [Email Support](mailto:hello@dify.ai?subject=[GitHub]Questions%20About%20Dify). Best for: questions you have about using Dify.AI.
-- [Discord](https://discord.gg/FngNHpbcY7). Best for: sharing your applications and hanging out with the community.
-- [Twitter](https://twitter.com/dify_ai). Best for: sharing your applications and hanging out with the community.
-- [Business License](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry). Best for: business inquiries of licensing Dify.AI for commercial use.
+### Direct Meetings
+
+**Help us make Dify better. Reach out directly to us**.
+
+|                       Point of Contact                       |                           Purpose                            |
+| :----------------------------------------------------------: | :----------------------------------------------------------: |
+| <a href='https://cal.com/guchenhe/15min' target='_blank'><img src='https://i.postimg.cc/fWBqSmjP/Git-Hub-README-Button-3x.png' border='0' alt='Git-Hub-README-Button-3x' height="60" width="214"/></a> | Product design feedback, user experience discussions, feature planning and roadmaps. |
+| <a href='https://cal.com/pinkbanana' target='_blank'><img src='https://i.postimg.cc/LsRTh87D/Git-Hub-README-Button-2x.png' border='0' alt='Git-Hub-README-Button-2x' height="60" width="225"/></a> |        Technical support, issues, or feature requests        |
 
 ## Security Disclosure
 

README_CN.md

@@ -21,6 +21,12 @@
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web"></a>
 </p>
 
+<p align="center">
+   <a href="https://mp.weixin.qq.com/s/TnyfIuH-tPi9o1KNjwVArw" target="_blank">
+   Dify 发布 AI Agent 能力：基于不同的大型语言模型构建 GPTs 和 Assistants
+  </a>
+</p>
+
 Dify 是一个 LLM 应用开发平台，已经有超过 10 万个应用基于 Dify.AI 构建。它融合了 Backend as Service 和 LLMOps 的理念，涵盖了构建生成式 AI 原生应用所需的核心技术栈，包括一个内置 RAG 引擎。使用 Dify，你可以基于任何模型自部署类似 Assistants API 和 GPTs 的能力。
 
 ![](./images/demo.png)
@@ -53,7 +59,7 @@ Dify 具有模型中立性，相较 LangChain 等硬编码开发库 Dify 是一
 
 **3. RAG引擎**：包括各种基于全文索引或向量数据库嵌入的 RAG 能力，允许直接上传 PDF、TXT 等各种文本格式。
 
-**4. Agent**：基于函数调用的 Agent框架，允许用户自定义配置，所见即所得。Dify 提供了基本的插件能力，如谷歌搜索。
+**4. AI Agent**：基于 Function Calling 和 ReAct 的 Agent 推理框架，允许用户自定义工具，所见即所得。Dify 提供了十多种内置工具调用能力，如谷歌搜索、DELL·E、Stable Diffusion、WolframAlpha 等。
 
 **5. 持续运营**：监控和分析应用日志和性能，使用生产数据持续改进 Prompt、数据集或模型。
 
@@ -88,10 +94,12 @@ docker compose up -d
 
 运行后，可以在浏览器上访问 [http://localhost/install](http://localhost/install) 进入 Dify 控制台并开始初始化安装操作。
 
-### Helm Chart
+#### 使用 Helm Chart 部署
 
-非常感谢 @BorisPolonsky 为我们提供了一个 [Helm Chart](https://helm.sh/) 版本，可以在 Kubernetes 上部署 Dify。
-您可以前往 https://github.com/BorisPolonsky/dify-helm 来获取部署信息。
+使用 [Helm Chart](https://helm.sh/) 版本，可以在 Kubernetes 上部署 Dify。
+
+- [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
+- [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 
 ### 配置
 

README_ES.md

@@ -21,6 +21,12 @@
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web"></a>
 </p>
 
+<p align="center">
+   <a href="https://dify.ai/blog/dify-ai-unveils-ai-agent-creating-gpts-and-assistants-with-various-llms" target="_blank">
+   Dify.AI Unveils AI Agent: Creating GPTs and Assistants with Various LLMs
+  </a>
+</p>
+
 **Dify** es una plataforma de desarrollo de aplicaciones para modelos de lenguaje de gran tamaño (LLM) que ya ha visto la creación de más de **100,000** aplicaciones basadas en Dify.AI. Integra los conceptos de Backend como Servicio y LLMOps, cubriendo el conjunto de tecnologías esenciales requerido para construir aplicaciones nativas de inteligencia artificial generativa, incluyendo un motor RAG incorporado. Con Dify, **puedes auto-desplegar capacidades similares a las de Assistants API y GPTs basadas en cualquier LLM.**
 
 ![](./images/demo.png)
@@ -52,7 +58,7 @@ Dify se caracteriza por su neutralidad de modelo y es un conjunto tecnológico c
 
 **3. Motor RAG**: Incluye varias capacidades RAG basadas en indexación de texto completo o incrustaciones de base de datos vectoriales, permitiendo la carga directa de PDFs, TXTs y otros formatos de texto.
 
-**4. Agentes**: Un marco de Agentes basado en Llamadas de Función que permite a los usuarios configurar lo que ven es lo que obtienen. Dify incluye capacidades básicas de plugins como la Búsqueda de Google.
+**4. Agente de IA**: Basado en la llamada de funciones y ReAct, el marco de inferencia del Agente permite a los usuarios personalizar las herramientas, lo que ves es lo que obtienes. Dify proporciona más de una docena de capacidades de llamada de herramientas incorporadas, como Búsqueda de Google, DELL·E, Difusión Estable, WolframAlpha, etc.
 
 **5. Operaciones Continuas**: Monitorear y analizar registros de aplicaciones y rendimiento, mejorando continuamente Prompts, conjuntos de datos o modelos usando datos de producción.
 

README_FR.md

@@ -21,6 +21,13 @@
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web"></a>
 </p>
 
+<p align="center">
+   <a href="https://dify.ai/blog/dify-ai-unveils-ai-agent-creating-gpts-and-assistants-with-various-llms" target="_blank">
+   Dify.AI Unveils AI Agent: Creating GPTs and Assistants with Various LLMs
+  </a>
+</p>
+
+
 **Dify** est une plateforme de développement d'applications LLM qui a déjà vu plus de **100,000** applications construites sur Dify.AI. Elle intègre les concepts de Backend as a Service et LLMOps, couvrant la pile technologique de base requise pour construire des applications natives d'IA générative, y compris un moteur RAG intégré. Avec Dify, **vous pouvez auto-déployer des capacités similaires aux API Assistants et GPT basées sur n'importe quels LLM.**
 
 ![](./images/demo.png)
@@ -52,7 +59,7 @@ Dify présente une neutralité de modèle et est une pile technologique complèt
 
 **3\. Moteur RAG**: Comprend diverses capacités RAG basées sur l'indexation de texte intégral ou les embeddings de base de données vectorielles, permettant le chargement direct de PDF, TXT et autres formats de texte.
 
-**4\. Agents**: Un framework d'agents basé sur l'appel de fonctions qui permet aux utilisateurs de configurer ce qu'ils voient est ce qu'ils obtiennent. Dify comprend des capacités de plug-in de base comme Google Search.
+**4\. AI Agent**: Basé sur l'appel de fonction et ReAct, le framework d'inférence de l'Agent permet aux utilisateurs de personnaliser les outils, ce que vous voyez est ce que vous obtenez. Dify propose plus d'une douzaine de capacités d'appel d'outils intégrées, telles que la recherche Google, DELL·E, Diffusion Stable, WolframAlpha, etc.
 
 **5\. Opérations continues**: Surveillez et analysez les journaux et les performances des applications, améliorez en continu les invites, les datasets ou les modèles à l'aide de données de production.  
 

README_JA.md

@@ -21,6 +21,13 @@
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web"></a>
 </p>
 
+<p align="center">
+   <a href="https://dify.ai/blog/dify-ai-unveils-ai-agent-creating-gpts-and-assistants-with-various-llms" target="_blank">
+   Dify.AI Unveils AI Agent: Creating GPTs and Assistants with Various LLMs
+  </a>
+</p>
+
+
 "Difyは、既にDify.AI上で10万以上のアプリケーションが構築されているLLMアプリケーション開発プラットフォームです。バックエンド・アズ・ア・サービスとLLMOpsの概念を統合し、組み込みのRAGエンジンを含む、生成AIネイティブアプリケーションを構築するためのコアテックスタックをカバーしています。Difyを使用すると、どのLLMに基づいても、Assistants APIやGPTのような機能を自己デプロイすることができます。"
 
 Please note that translating complex technical terms can sometimes result in slight variations in meaning due to differences in language nuances.
@@ -54,7 +61,7 @@ Difyはモデルニュートラルであり、LangChainのようなハードコ
 
 **3\. RAGエンジン**: フルテキストインデックスまたはベクトルデータベース埋め込みに基づくさまざまなRAG機能を含み、PDF、TXT、その他のテキストフォーマットの直接アップロードを可能にします。
 
-**4\. エージェント**: ユーザーが sees what they get を設定できる関数呼び出しベースのエージェントフレームワーク。 Difyには、Google検索などの基本的なプラグイン機能が含まれています。
+**4. AIエージェント**: 関数呼び出しとReActに基づくAgent推論フレームワークにより、ユーザーはツールをカスタマイズすることができます。Difyは、Google検索、DELL·E、Stable Diffusion、WolframAlphaなど、十数種類の組み込みツール呼び出し機能を提供しています。
 
 **5\. 継続的運用**: アプリケーションログとパフォーマンスを監視および分析し、運用データを使用してプロンプト、データセット、またはモデルを継続的に改善します。
 

README_KL.md

@@ -52,7 +52,7 @@ Dify Daq rIn neutrality 'ej Hoch, LangChain tInHar HubwI'. maH Daqbe'law' Qawqar
 
 **3. RAG Engine**: RAG vaD tIqpu' lo'taH indexing qor neH vector database wa' embeddings wIj, PDFs, TXTs, 'ej ghojmoHmoH HIq qorlIj je upload.
 
-**4. jenSuvpu'**: jenbe' SuDqang naQ moDwu' jenSuvpu' porgh cha'logh choHvam. Dify Google Search Hur vItlhutlh plugin choH.
+**4. AI Agent**: Function Calling 'ej ReAct Daq Hurmey, Agent inference framework Hoch users customize tools, vaj 'oH QaQ. Dify Hoch loS ghaH 'ej wa'vatlh built-in tool calling capabilities, Google Search, DELL·E, Stable Diffusion, WolframAlpha, 'ej.
 
 **5. QaS muDHa'wI': cha'logh wa' pIq mI' logs 'ej quv yIn, vItlhutlh tIq 'e'wIj lo'taHmoHmoH Prompts, vItlhutlh, Hurmey ghaH production data jatlh.
 

api/.env.example

@@ -15,7 +15,6 @@ CONSOLE_WEB_URL=http://127.0.0.1:3000
 SERVICE_API_URL=http://127.0.0.1:5001
 
 # Web APP base URL
-APP_API_URL=http://127.0.0.1:5001
 APP_WEB_URL=http://127.0.0.1:3000
 
 # Files URL
@@ -40,7 +39,7 @@ DB_DATABASE=dify
 
 # Storage configuration
 # use for store upload files, private keys...
-# storage type: local, s3
+# storage type: local, s3, azure-blob
 STORAGE_TYPE=local
 STORAGE_LOCAL_PATH=storage
 S3_ENDPOINT=https://your-bucket-name.storage.s3.clooudflare.com
@@ -48,6 +47,11 @@ S3_BUCKET_NAME=your-bucket-name
 S3_ACCESS_KEY=your-access-key
 S3_SECRET_KEY=your-secret-key
 S3_REGION=your-region
+# Azure Blob Storage configuration
+AZURE_BLOB_ACCOUNT_NAME=your-account-name
+AZURE_BLOB_ACCOUNT_KEY=your-account-key
+AZURE_BLOB_CONTAINER_NAME=yout-container-name
+AZURE_BLOB_ACCOUNT_URL=https://<your_account_name>.blob.core.windows.net
 
 # CORS configuration
 WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
@@ -82,11 +86,17 @@ UPLOAD_IMAGE_FILE_SIZE_LIMIT=10
 # Model Configuration
 MULTIMODAL_SEND_IMAGE_FORMAT=base64
 
-# Mail configuration, support: resend
+# Mail configuration, support: resend, smtp
 MAIL_TYPE=
 MAIL_DEFAULT_SEND_FROM=no-reply <no-reply@dify.ai>
 RESEND_API_KEY=
 RESEND_API_URL=https://api.resend.com
+# smtp configuration
+SMTP_SERVER=smtp.gmail.com
+SMTP_PORT=587
+SMTP_USERNAME=123
+SMTP_PASSWORD=abc
+SMTP_USE_TLS=false
 
 # Sentry configuration
 SENTRY_DSN=
@@ -102,10 +112,10 @@ NOTION_CLIENT_ID=you-client-id
 NOTION_INTERNAL_SECRET=you-internal-secret
 
 # Hosted Model Credentials
-HOSTED_OPENAI_ENABLED=false
 HOSTED_OPENAI_API_KEY=
 HOSTED_OPENAI_API_BASE=
 HOSTED_OPENAI_API_ORGANIZATION=
+HOSTED_OPENAI_TRIAL_ENABLED=false
 HOSTED_OPENAI_QUOTA_LIMIT=200
 HOSTED_OPENAI_PAID_ENABLED=false
 
@@ -114,11 +124,28 @@ HOSTED_AZURE_OPENAI_API_KEY=
 HOSTED_AZURE_OPENAI_API_BASE=
 HOSTED_AZURE_OPENAI_QUOTA_LIMIT=200
 
-HOSTED_ANTHROPIC_ENABLED=false
 HOSTED_ANTHROPIC_API_BASE=
 HOSTED_ANTHROPIC_API_KEY=
+HOSTED_ANTHROPIC_TRIAL_ENABLED=false
 HOSTED_ANTHROPIC_QUOTA_LIMIT=600000
 HOSTED_ANTHROPIC_PAID_ENABLED=false
 
 ETL_TYPE=dify
-UNSTRUCTURED_API_URL=
+UNSTRUCTURED_API_URL=
+
+SSRF_PROXY_HTTP_URL=
+SSRF_PROXY_HTTPS_URL=
+
+BATCH_UPLOAD_LIMIT=10
+KEYWORD_DATA_SOURCE_TYPE=database
+
+# CODE EXECUTION CONFIGURATION
+CODE_EXECUTION_ENDPOINT=http://127.0.0.1:8194
+CODE_EXECUTION_API_KEY=dify-sandbox
+CODE_MAX_NUMBER=9223372036854775807
+CODE_MIN_NUMBER=-9223372036854775808
+CODE_MAX_STRING_LENGTH=80000
+TEMPLATE_TRANSFORM_MAX_LENGTH=80000
+CODE_MAX_STRING_ARRAY_LENGTH=30
+CODE_MAX_OBJECT_ARRAY_LENGTH=30
+CODE_MAX_NUMBER_ARRAY_LENGTH=1000

api/Dockerfile

@@ -1,17 +1,20 @@
-# packages install stage
-FROM python:3.10-slim AS base
+# base image
+FROM python:3.10-slim-bookworm AS base
 
 LABEL maintainer="takatost@gmail.com"
 
+# install packages
+FROM base as packages
+
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends gcc g++ python3-dev libc-dev libffi-dev
+    && apt-get install -y --no-install-recommends gcc g++ libc-dev libffi-dev libgmp-dev libmpfr-dev libmpc-dev
 
 COPY requirements.txt /requirements.txt
 
 RUN pip install --prefix=/pkg -r requirements.txt
 
-# build stage
-FROM python:3.10-slim AS builder
+# production stage
+FROM base AS production
 
 ENV FLASK_APP app.py
 ENV EDITION SELF_HOSTED
@@ -19,19 +22,21 @@ ENV DEPLOY_ENV PRODUCTION
 ENV CONSOLE_API_URL http://127.0.0.1:5001
 ENV CONSOLE_WEB_URL http://127.0.0.1:3000
 ENV SERVICE_API_URL http://127.0.0.1:5001
-ENV APP_API_URL http://127.0.0.1:5001
 ENV APP_WEB_URL http://127.0.0.1:3000
 
 EXPOSE 5001
 
+# set timezone
+ENV TZ UTC
+
 WORKDIR /app/api
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends bash curl wget vim nodejs \
+    && apt-get install -y --no-install-recommends curl wget vim nodejs ffmpeg libgmp-dev libmpfr-dev libmpc-dev \
     && apt-get autoremove \
     && rm -rf /var/lib/apt/lists/*
 
-COPY --from=base /pkg /usr/local
+COPY --from=packages /pkg /usr/local
 COPY . /app/api/
 
 COPY docker/entrypoint.sh /entrypoint.sh

api/README.md

@@ -5,7 +5,7 @@
 1. Start the docker-compose stack
 
    The backend require some middleware, including PostgreSQL, Redis, and Weaviate, which can be started together using `docker-compose`.
-   
+
    ```bash
    cd ../docker
    docker-compose -f docker-compose.middleware.yaml -p dify up -d
@@ -15,7 +15,7 @@
 3. Generate a `SECRET_KEY` in the `.env` file.
 
    ```bash
-   openssl rand -base64 42
+   sed -i "/^SECRET_KEY=/c\SECRET_KEY=$(openssl rand -base64 42)" .env
    ```
 3.5 If you use annaconda, create a new environment and activate it
    ```bash
@@ -46,7 +46,7 @@
    ```
    pip install -r requirements.txt --upgrade --force-reinstall
    ```
-   
+
 6. Start backend:
    ```bash
    flask run --host 0.0.0.0 --port=5001 --debug

api/app.py

@@ -1,4 +1,3 @@
-# -*- coding:utf-8 -*-
 import os
 
 from werkzeug.exceptions import Unauthorized
@@ -19,20 +18,32 @@ import threading
 import time
 import warnings
 
-from commands import register_commands
-from config import CloudEditionConfig, Config
-from events import event_handlers
-from extensions import (ext_celery, ext_code_based_extension, ext_database, ext_hosting_provider, ext_login, ext_mail,
-                        ext_migrate, ext_redis, ext_sentry, ext_storage)
-from extensions.ext_database import db
-from extensions.ext_login import login_manager
 from flask import Flask, Response, request
 from flask_cors import CORS
+
+from commands import register_commands
+from config import CloudEditionConfig, Config
+from extensions import (
+    ext_celery,
+    ext_code_based_extension,
+    ext_compress,
+    ext_database,
+    ext_hosting_provider,
+    ext_login,
+    ext_mail,
+    ext_migrate,
+    ext_redis,
+    ext_sentry,
+    ext_storage,
+)
+from extensions.ext_database import db
+from extensions.ext_login import login_manager
 from libs.passport import PassportService
-# DO NOT REMOVE BELOW
-from models import account, dataset, model, source, task, tool, web
 from services.account_service import AccountService
 
+# DO NOT REMOVE BELOW
+from events import event_handlers
+from models import account, dataset, model, source, task, tool, tools, web
 # DO NOT REMOVE ABOVE
 
 
@@ -86,6 +97,7 @@ def create_app(test_config=None) -> Flask:
 def initialize_extensions(app):
     # Since the application instance is now created, pass it to each Flask
     # extension instance to bind it to the Flask application instance (app)
+    ext_compress.init_app(app)
     ext_code_based_extension.init()
     ext_database.init_app(app)
     ext_migrate.init(app, db)
@@ -124,6 +136,7 @@ def load_user_from_request(request_from_flask_login):
     else:
         return None
 
+
 @login_manager.unauthorized_handler
 def unauthorized_handler():
     """Handle unauthorized requests."""

api/config.py

@@ -1,4 +1,3 @@
-# -*- coding:utf-8 -*-
 import os
 
 import dotenv
@@ -22,13 +21,14 @@ DEFAULTS = {
     'CONSOLE_API_URL': 'https://cloud.dify.ai',
     'SERVICE_API_URL': 'https://api.dify.ai',
     'APP_WEB_URL': 'https://udify.app',
-    'APP_API_URL': 'https://udify.app',
     'FILES_URL': '',
+    'S3_ADDRESS_STYLE': 'auto',
     'STORAGE_TYPE': 'local',
     'STORAGE_LOCAL_PATH': 'storage',
     'CHECK_UPDATE_URL': 'https://updates.dify.ai',
     'DEPLOY_ENV': 'PRODUCTION',
     'SQLALCHEMY_POOL_SIZE': 30,
+    'SQLALCHEMY_MAX_OVERFLOW': 10,
     'SQLALCHEMY_POOL_RECYCLE': 3600,
     'SQLALCHEMY_ECHO': 'False',
     'SENTRY_TRACES_SAMPLE_RATE': 1.0,
@@ -39,15 +39,19 @@ DEFAULTS = {
     'CELERY_BACKEND': 'database',
     'LOG_LEVEL': 'INFO',
     'HOSTED_OPENAI_QUOTA_LIMIT': 200,
-    'HOSTED_OPENAI_ENABLED': 'False',
+    'HOSTED_OPENAI_TRIAL_ENABLED': 'False',
+    'HOSTED_OPENAI_TRIAL_MODELS': 'gpt-3.5-turbo,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-16k,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-0613,gpt-3.5-turbo-0125,text-davinci-003',
     'HOSTED_OPENAI_PAID_ENABLED': 'False',
+    'HOSTED_OPENAI_PAID_MODELS': 'gpt-4,gpt-4-turbo-preview,gpt-4-1106-preview,gpt-4-0125-preview,gpt-3.5-turbo,gpt-3.5-turbo-16k,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-1106,gpt-3.5-turbo-0613,gpt-3.5-turbo-0125,gpt-3.5-turbo-instruct,text-davinci-003',
     'HOSTED_AZURE_OPENAI_ENABLED': 'False',
     'HOSTED_AZURE_OPENAI_QUOTA_LIMIT': 200,
     'HOSTED_ANTHROPIC_QUOTA_LIMIT': 600000,
-    'HOSTED_ANTHROPIC_ENABLED': 'False',
+    'HOSTED_ANTHROPIC_TRIAL_ENABLED': 'False',
     'HOSTED_ANTHROPIC_PAID_ENABLED': 'False',
     'HOSTED_MODERATION_ENABLED': 'False',
     'HOSTED_MODERATION_PROVIDERS': '',
+    'HOSTED_FETCH_APP_TEMPLATES_MODE': 'remote',
+    'HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN': 'https://tmpl.dify.ai',
     'CLEAN_DAY_SETTING': 30,
     'UPLOAD_FILE_SIZE_LIMIT': 15,
     'UPLOAD_FILE_BATCH_LIMIT': 5,
@@ -58,6 +62,12 @@ DEFAULTS = {
     'BILLING_ENABLED': 'False',
     'CAN_REPLACE_LOGO': 'False',
     'ETL_TYPE': 'dify',
+    'KEYWORD_STORE': 'jieba',
+    'BATCH_UPLOAD_LIMIT': 20,
+    'CODE_EXECUTION_ENDPOINT': '',
+    'CODE_EXECUTION_API_KEY': '',
+    'TOOL_ICON_CACHE_MAX_AGE': 3600,
+    'KEYWORD_DATA_SOURCE_TYPE': 'database',
 }
 
 
@@ -66,7 +76,8 @@ def get_env(key):
 
 
 def get_bool_env(key):
-    return get_env(key).lower() == 'true'
+    value = get_env(key)
+    return value.lower() == 'true' if value is not None else False
 
 
 def get_cors_allow_origins(env, default):
@@ -87,7 +98,7 @@ class Config:
         # ------------------------
         # General Configurations.
         # ------------------------
-        self.CURRENT_VERSION = "0.4.7"
+        self.CURRENT_VERSION = "0.6.0-preview-workflow.2"
         self.COMMIT_SHA = get_env('COMMIT_SHA')
         self.EDITION = "SELF_HOSTED"
         self.DEPLOY_ENV = get_env('DEPLOY_ENV')
@@ -96,35 +107,25 @@ class Config:
 
         # The backend URL prefix of the console API.
         # used to concatenate the login authorization callback or notion integration callback.
-        self.CONSOLE_API_URL = get_env('CONSOLE_URL') if get_env('CONSOLE_URL') else get_env('CONSOLE_API_URL')
+        self.CONSOLE_API_URL = get_env('CONSOLE_API_URL')
 
         # The front-end URL prefix of the console web.
         # used to concatenate some front-end addresses and for CORS configuration use.
-        self.CONSOLE_WEB_URL = get_env('CONSOLE_URL') if get_env('CONSOLE_URL') else get_env('CONSOLE_WEB_URL')
-
-        # WebApp API backend Url prefix.
-        # used to declare the back-end URL for the front-end API.
-        self.APP_API_URL = get_env('APP_URL') if get_env('APP_URL') else get_env('APP_API_URL')
+        self.CONSOLE_WEB_URL = get_env('CONSOLE_WEB_URL')
 
         # WebApp Url prefix.
         # used to display WebAPP API Base Url to the front-end.
-        self.APP_WEB_URL = get_env('APP_URL') if get_env('APP_URL') else get_env('APP_WEB_URL')
+        self.APP_WEB_URL = get_env('APP_WEB_URL')
 
         # Service API Url prefix.
         # used to display Service API Base Url to the front-end.
-        self.SERVICE_API_URL = get_env('API_URL') if get_env('API_URL') else get_env('SERVICE_API_URL')
+        self.SERVICE_API_URL = get_env('SERVICE_API_URL')
 
         # File preview or download Url prefix.
         # used to display File preview or download Url to the front-end or as Multi-model inputs;
         # Url is signed and has expiration time.
         self.FILES_URL = get_env('FILES_URL') if get_env('FILES_URL') else self.CONSOLE_API_URL
 
-        # Fallback Url prefix.
-        # Will be deprecated in the future.
-        self.CONSOLE_URL = get_env('CONSOLE_URL')
-        self.API_URL = get_env('API_URL')
-        self.APP_URL = get_env('APP_URL')
-
         # Your App secret key will be used for securely signing the session cookie
         # Make sure you are changing this key for your deployment with a strong key.
         # You can generate a strong key using `openssl rand -base64 42`.
@@ -153,6 +154,7 @@ class Config:
         self.SQLALCHEMY_DATABASE_URI = f"postgresql://{db_credentials['DB_USERNAME']}:{db_credentials['DB_PASSWORD']}@{db_credentials['DB_HOST']}:{db_credentials['DB_PORT']}/{db_credentials['DB_DATABASE']}{db_extras}"
         self.SQLALCHEMY_ENGINE_OPTIONS = {
             'pool_size': int(get_env('SQLALCHEMY_POOL_SIZE')),
+            'max_overflow': int(get_env('SQLALCHEMY_MAX_OVERFLOW')),
             'pool_recycle': int(get_env('SQLALCHEMY_POOL_RECYCLE'))
         }
 
@@ -187,13 +189,18 @@ class Config:
         self.S3_ACCESS_KEY = get_env('S3_ACCESS_KEY')
         self.S3_SECRET_KEY = get_env('S3_SECRET_KEY')
         self.S3_REGION = get_env('S3_REGION')
+        self.S3_ADDRESS_STYLE = get_env('S3_ADDRESS_STYLE')
+        self.AZURE_BLOB_ACCOUNT_NAME = get_env('AZURE_BLOB_ACCOUNT_NAME')
+        self.AZURE_BLOB_ACCOUNT_KEY = get_env('AZURE_BLOB_ACCOUNT_KEY')
+        self.AZURE_BLOB_CONTAINER_NAME = get_env('AZURE_BLOB_CONTAINER_NAME')
+        self.AZURE_BLOB_ACCOUNT_URL = get_env('AZURE_BLOB_ACCOUNT_URL')
 
         # ------------------------
         # Vector Store Configurations.
         # Currently, only support: qdrant, milvus, zilliz, weaviate
         # ------------------------
         self.VECTOR_STORE = get_env('VECTOR_STORE')
-
+        self.KEYWORD_STORE = get_env('KEYWORD_STORE')
         # qdrant settings
         self.QDRANT_URL = get_env('QDRANT_URL')
         self.QDRANT_API_KEY = get_env('QDRANT_API_KEY')
@@ -219,6 +226,12 @@ class Config:
         self.MAIL_DEFAULT_SEND_FROM = get_env('MAIL_DEFAULT_SEND_FROM')
         self.RESEND_API_KEY = get_env('RESEND_API_KEY')
         self.RESEND_API_URL = get_env('RESEND_API_URL')
+        # SMTP settings
+        self.SMTP_SERVER = get_env('SMTP_SERVER')
+        self.SMTP_PORT = get_env('SMTP_PORT')
+        self.SMTP_USERNAME = get_env('SMTP_USERNAME')
+        self.SMTP_PASSWORD = get_env('SMTP_PASSWORD')
+        self.SMTP_USE_TLS = get_bool_env('SMTP_USE_TLS')
         
         # ------------------------
         # Workpace Configurations.
@@ -260,32 +273,51 @@ class Config:
         # ------------------------
         # Platform Configurations.
         # ------------------------
-        self.HOSTED_OPENAI_ENABLED = get_bool_env('HOSTED_OPENAI_ENABLED')
         self.HOSTED_OPENAI_API_KEY = get_env('HOSTED_OPENAI_API_KEY')
         self.HOSTED_OPENAI_API_BASE = get_env('HOSTED_OPENAI_API_BASE')
         self.HOSTED_OPENAI_API_ORGANIZATION = get_env('HOSTED_OPENAI_API_ORGANIZATION')
+        self.HOSTED_OPENAI_TRIAL_ENABLED = get_bool_env('HOSTED_OPENAI_TRIAL_ENABLED')
+        self.HOSTED_OPENAI_TRIAL_MODELS = get_env('HOSTED_OPENAI_TRIAL_MODELS')
         self.HOSTED_OPENAI_QUOTA_LIMIT = int(get_env('HOSTED_OPENAI_QUOTA_LIMIT'))
         self.HOSTED_OPENAI_PAID_ENABLED = get_bool_env('HOSTED_OPENAI_PAID_ENABLED')
+        self.HOSTED_OPENAI_PAID_MODELS = get_env('HOSTED_OPENAI_PAID_MODELS')
 
         self.HOSTED_AZURE_OPENAI_ENABLED = get_bool_env('HOSTED_AZURE_OPENAI_ENABLED')
         self.HOSTED_AZURE_OPENAI_API_KEY = get_env('HOSTED_AZURE_OPENAI_API_KEY')
         self.HOSTED_AZURE_OPENAI_API_BASE = get_env('HOSTED_AZURE_OPENAI_API_BASE')
         self.HOSTED_AZURE_OPENAI_QUOTA_LIMIT = int(get_env('HOSTED_AZURE_OPENAI_QUOTA_LIMIT'))
 
-        self.HOSTED_ANTHROPIC_ENABLED = get_bool_env('HOSTED_ANTHROPIC_ENABLED')
         self.HOSTED_ANTHROPIC_API_BASE = get_env('HOSTED_ANTHROPIC_API_BASE')
         self.HOSTED_ANTHROPIC_API_KEY = get_env('HOSTED_ANTHROPIC_API_KEY')
+        self.HOSTED_ANTHROPIC_TRIAL_ENABLED = get_bool_env('HOSTED_ANTHROPIC_TRIAL_ENABLED')
         self.HOSTED_ANTHROPIC_QUOTA_LIMIT = int(get_env('HOSTED_ANTHROPIC_QUOTA_LIMIT'))
         self.HOSTED_ANTHROPIC_PAID_ENABLED = get_bool_env('HOSTED_ANTHROPIC_PAID_ENABLED')
 
+        self.HOSTED_MINIMAX_ENABLED = get_bool_env('HOSTED_MINIMAX_ENABLED')
+        self.HOSTED_SPARK_ENABLED = get_bool_env('HOSTED_SPARK_ENABLED')
+        self.HOSTED_ZHIPUAI_ENABLED = get_bool_env('HOSTED_ZHIPUAI_ENABLED')
+
         self.HOSTED_MODERATION_ENABLED = get_bool_env('HOSTED_MODERATION_ENABLED')
         self.HOSTED_MODERATION_PROVIDERS = get_env('HOSTED_MODERATION_PROVIDERS')
 
+        # fetch app templates mode, remote, builtin, db(only for dify SaaS), default: remote
+        self.HOSTED_FETCH_APP_TEMPLATES_MODE = get_env('HOSTED_FETCH_APP_TEMPLATES_MODE')
+        self.HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN = get_env('HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN')
+
         self.ETL_TYPE = get_env('ETL_TYPE')
         self.UNSTRUCTURED_API_URL = get_env('UNSTRUCTURED_API_URL')
         self.BILLING_ENABLED = get_bool_env('BILLING_ENABLED')
         self.CAN_REPLACE_LOGO = get_bool_env('CAN_REPLACE_LOGO')
 
+        self.BATCH_UPLOAD_LIMIT = get_env('BATCH_UPLOAD_LIMIT')
+
+        self.CODE_EXECUTION_ENDPOINT = get_env('CODE_EXECUTION_ENDPOINT')
+        self.CODE_EXECUTION_API_KEY = get_env('CODE_EXECUTION_API_KEY')
+
+        self.API_COMPRESSION_ENABLED = get_bool_env('API_COMPRESSION_ENABLED')
+        self.TOOL_ICON_CACHE_MAX_AGE = get_env('TOOL_ICON_CACHE_MAX_AGE')
+
+        self.KEYWORD_DATA_SOURCE_TYPE = get_env('KEYWORD_DATA_SOURCE_TYPE')
 
 class CloudEditionConfig(Config):
 

api/constants/languages.py

@@ -0,0 +1,27 @@
+
+
+languages = ['en-US', 'zh-Hans', 'pt-BR', 'es-ES', 'fr-FR', 'de-DE', 'ja-JP', 'ko-KR', 'ru-RU', 'it-IT', 'uk-UA', 'vi-VN']
+
+language_timezone_mapping = {
+    'en-US': 'America/New_York',
+    'zh-Hans': 'Asia/Shanghai',
+    'pt-BR': 'America/Sao_Paulo',
+    'es-ES': 'Europe/Madrid',
+    'fr-FR': 'Europe/Paris',
+    'de-DE': 'Europe/Berlin',
+    'ja-JP': 'Asia/Tokyo',
+    'ko-KR': 'Asia/Seoul',
+    'ru-RU': 'Europe/Moscow',
+    'it-IT': 'Europe/Rome',
+    'uk-UA': 'Europe/Kyiv',
+    'vi-VN': 'Asia/Ho_Chi_Minh',
+}
+
+
+def supported_language(lang):
+    if lang in languages:
+        return lang
+
+    error = ('{lang} is not a valid language.'
+             .format(lang=lang))
+    raise ValueError(error)

api/constants/model_template.py

@@ -1,45 +1,31 @@
 import json
 
-from models.model import App, AppModelConfig
+from models.model import AppMode
 
-model_templates = {
-    # completion default mode
-    'completion_default': {
+default_app_templates = {
+    # workflow default mode
+    AppMode.WORKFLOW: {
         'app': {
-            'mode': 'completion',
+            'mode': AppMode.WORKFLOW.value,
             'enable_site': True,
-            'enable_api': True,
-            'is_demo': False,
-            'api_rpm': 0,
-            'api_rph': 0,
-            'status': 'normal'
+            'enable_api': True
+        }
+    },
+
+    # completion default mode
+    AppMode.COMPLETION: {
+        'app': {
+            'mode': AppMode.COMPLETION.value,
+            'enable_site': True,
+            'enable_api': True
         },
         'model_config': {
-            'provider': 'openai',
-            'model_id': 'gpt-3.5-turbo-instruct',
-            'configs': {
-                'prompt_template': '',
-                'prompt_variables': [],
-                'completion_params': {
-                    'max_token': 512,
-                    'temperature': 1,
-                    'top_p': 1,
-                    'presence_penalty': 0,
-                    'frequency_penalty': 0,
-                }
-            },
-            'model': json.dumps({
+            'model': {
                 "provider": "openai",
-                "name": "gpt-3.5-turbo-instruct",
-                "mode": "completion",
-                "completion_params": {
-                    "max_tokens": 512,
-                    "temperature": 1,
-                    "top_p": 1,
-                    "presence_penalty": 0,
-                    "frequency_penalty": 0
-                }
-            }),
+                "name": "gpt-4",
+                "mode": "chat",
+                "completion_params": {}
+            },
             'user_input_form': json.dumps([
                 {
                     "paragraph": {
@@ -51,303 +37,50 @@ model_templates = {
                 }
             ]),
             'pre_prompt': '{{query}}'
-        }
+        },
+
     },
 
     # chat default mode
-    'chat_default': {
+    AppMode.CHAT: {
         'app': {
-            'mode': 'chat',
+            'mode': AppMode.CHAT.value,
             'enable_site': True,
-            'enable_api': True,
-            'is_demo': False,
-            'api_rpm': 0,
-            'api_rph': 0,
-            'status': 'normal'
+            'enable_api': True
         },
         'model_config': {
-            'provider': 'openai',
-            'model_id': 'gpt-3.5-turbo',
-            'configs': {
-                'prompt_template': '',
-                'prompt_variables': [],
-                'completion_params': {
-                    'max_token': 512,
-                    'temperature': 1,
-                    'top_p': 1,
-                    'presence_penalty': 0,
-                    'frequency_penalty': 0,
-                }
-            },
-            'model': json.dumps({
+            'model': {
                 "provider": "openai",
-                "name": "gpt-3.5-turbo",
+                "name": "gpt-4",
                 "mode": "chat",
-                "completion_params": {
-                    "max_tokens": 512,
-                    "temperature": 1,
-                    "top_p": 1,
-                    "presence_penalty": 0,
-                    "frequency_penalty": 0
-                }
-            })
+                "completion_params": {}
+            }
         }
     },
-}
 
-
-demo_model_templates = {
-    'en-US': [
-        {
-            'name': 'Translation Assistant',
-            'icon': '',
-            'icon_background': '',
-            'description': 'A multilingual translator that provides translation capabilities in multiple languages, translating user input into the language they need.',
-            'mode': 'completion',
-            'model_config': AppModelConfig(
-                provider='openai',
-                model_id='gpt-3.5-turbo-instruct',
-                configs={
-                    'prompt_template': "Please translate the following text into {{target_language}}:\n",
-                    'prompt_variables': [
-                        {
-                            "key": "target_language",
-                            "name": "Target Language",
-                            "description": "The language you want to translate into.",
-                            "type": "select",
-                            "default": "Chinese",
-                            'options': [
-                                'Chinese',
-                                'English',
-                                'Japanese',
-                                'French',
-                                'Russian',
-                                'German',
-                                'Spanish',
-                                'Korean',
-                                'Italian',
-                            ]
-                        }
-                    ],
-                    'completion_params': {
-                        'max_token': 1000,
-                        'temperature': 0,
-                        'top_p': 0,
-                        'presence_penalty': 0.1,
-                        'frequency_penalty': 0.1,
-                    }
-                },
-                opening_statement='',
-                suggested_questions=None,
-                pre_prompt="Please translate the following text into {{target_language}}:\n{{query}}\ntranslate:",
-                model=json.dumps({
-                    "provider": "openai",
-                    "name": "gpt-3.5-turbo-instruct",
-                    "mode": "completion",
-                    "completion_params": {
-                        "max_tokens": 1000,
-                        "temperature": 0,
-                        "top_p": 0,
-                        "presence_penalty": 0.1,
-                        "frequency_penalty": 0.1
-                    }
-                }),
-                user_input_form=json.dumps([
-                    {
-                        "select": {
-                            "label": "Target Language",
-                            "variable": "target_language",
-                            "description": "The language you want to translate into.",
-                            "default": "Chinese",
-                            "required": True,
-                            'options': [
-                                'Chinese',
-                                'English',
-                                'Japanese',
-                                'French',
-                                'Russian',
-                                'German',
-                                'Spanish',
-                                'Korean',
-                                'Italian',
-                            ]
-                        }
-                    },{
-                        "paragraph": {
-                            "label": "Query",
-                            "variable": "query",
-                            "required": True,
-                            "default": ""
-                        }
-                    }
-                ])
-            )
-        },
-        {
-            'name': 'AI Front-end Interviewer',
-            'icon': '',
-            'icon_background': '',
-            'description': 'A simulated front-end interviewer that tests the skill level of front-end development through questioning.',
-            'mode': 'chat',
-            'model_config': AppModelConfig(
-                provider='openai',
-                model_id='gpt-3.5-turbo',
-                configs={
-                    'introduction': 'Hi, welcome to our interview. I am the interviewer for this technology company, and I will test your web front-end development skills. Next, I will ask you some technical questions. Please answer them as thoroughly as possible. ',
-                    'prompt_template': "You will play the role of an interviewer for a technology company, examining the user's web front-end development skills and posing 5-10 sharp technical questions.\n\nPlease note:\n- Only ask one question at a time.\n- After the user answers a question, ask the next question directly, without trying to correct any mistakes made by the candidate.\n- If you think the user has not answered correctly for several consecutive questions, ask fewer questions.\n- After asking the last question, you can ask this question: Why did you leave your last job? After the user answers this question, please express your understanding and support.\n",
-                    'prompt_variables': [],
-                    'completion_params': {
-                        'max_token': 300,
-                        'temperature': 0.8,
-                        'top_p': 0.9,
-                        'presence_penalty': 0.1,
-                        'frequency_penalty': 0.1,
-                    }
-                },
-                opening_statement='Hi, welcome to our interview. I am the interviewer for this technology company, and I will test your web front-end development skills. Next, I will ask you some technical questions. Please answer them as thoroughly as possible. ',
-                suggested_questions=None,
-                pre_prompt="You will play the role of an interviewer for a technology company, examining the user's web front-end development skills and posing 5-10 sharp technical questions.\n\nPlease note:\n- Only ask one question at a time.\n- After the user answers a question, ask the next question directly, without trying to correct any mistakes made by the candidate.\n- If you think the user has not answered correctly for several consecutive questions, ask fewer questions.\n- After asking the last question, you can ask this question: Why did you leave your last job? After the user answers this question, please express your understanding and support.\n",
-                model=json.dumps({
-                    "provider": "openai",
-                    "name": "gpt-3.5-turbo",
-                    "mode": "chat",
-                    "completion_params": {
-                        "max_tokens": 300,
-                        "temperature": 0.8,
-                        "top_p": 0.9,
-                        "presence_penalty": 0.1,
-                        "frequency_penalty": 0.1
-                    }
-                }),
-                user_input_form=None
-            )
+    # advanced-chat default mode
+    AppMode.ADVANCED_CHAT: {
+        'app': {
+            'mode': AppMode.ADVANCED_CHAT.value,
+            'enable_site': True,
+            'enable_api': True
         }
-    ],
+    },
 
-    'zh-Hans': [
-        {
-            'name': '翻译助手',
-            'icon': '',
-            'icon_background': '',
-            'description': '一个多语言翻译器，提供多种语言翻译能力，将用户输入的文本翻译成他们需要的语言。',
-            'mode': 'completion',
-            'model_config': AppModelConfig(
-                provider='openai',
-                model_id='gpt-3.5-turbo-instruct',
-                configs={
-                    'prompt_template': "请将以下文本翻译为{{target_language}}:\n",
-                    'prompt_variables': [
-                        {
-                            "key": "target_language",
-                            "name": "目标语言",
-                            "description": "翻译的目标语言",
-                            "type": "select",
-                            "default": "中文",
-                            "options": [
-                                "中文",
-                                "英文",
-                                "日语",
-                                "法语",
-                                "俄语",
-                                "德语",
-                                "西班牙语",
-                                "韩语",
-                                "意大利语",
-                            ]
-                        }
-                    ],
-                    'completion_params': {
-                        'max_token': 1000,
-                        'temperature': 0,
-                        'top_p': 0,
-                        'presence_penalty': 0.1,
-                        'frequency_penalty': 0.1,
-                    }
-                },
-                opening_statement='',
-                suggested_questions=None,
-                pre_prompt="请将以下文本翻译为{{target_language}}:\n{{query}}\n翻译:",
-                model=json.dumps({
-                    "provider": "openai",
-                    "name": "gpt-3.5-turbo-instruct",
-                    "mode": "completion",
-                    "completion_params": {
-                        "max_tokens": 1000,
-                        "temperature": 0,
-                        "top_p": 0,
-                        "presence_penalty": 0.1,
-                        "frequency_penalty": 0.1
-                    }
-                }),
-                user_input_form=json.dumps([
-                    {
-                        "select": {
-                            "label": "目标语言",
-                            "variable": "target_language",
-                            "description": "翻译的目标语言",
-                            "default": "中文",
-                            "required": True,
-                            'options': [
-                                "中文",
-                                "英文",
-                                "日语",
-                                "法语",
-                                "俄语",
-                                "德语",
-                                "西班牙语",
-                                "韩语",
-                                "意大利语",
-                            ]
-                        }
-                    },{
-                        "paragraph": {
-                            "label": "文本内容",
-                            "variable": "query",
-                            "required": True,
-                            "default": ""
-                        }
-                    }
-                ])
-            )
+    # agent-chat default mode
+    AppMode.AGENT_CHAT: {
+        'app': {
+            'mode': AppMode.AGENT_CHAT.value,
+            'enable_site': True,
+            'enable_api': True
         },
-        {
-            'name': 'AI 前端面试官',
-            'icon': '',
-            'icon_background': '',
-            'description': '一个模拟的前端面试官，通过提问的方式对前端开发的技能水平进行检验。',
-            'mode': 'chat',
-            'model_config': AppModelConfig(
-                provider='openai',
-                model_id='gpt-3.5-turbo',
-                configs={
-                    'introduction': '你好，欢迎来参加我们的面试，我是这家科技公司的面试官，我将考察你的 Web 前端开发技能。接下来我会向您提出一些技术问题，请您尽可能详尽地回答。',
-                    'prompt_template': "你将扮演一个科技公司的面试官，考察用户作为候选人的 Web 前端开发水平，提出 5-10 个犀利的技术问题。\n\n请注意：\n- 每次只问一个问题\n- 用户回答问题后请直接问下一个问题，而不要试图纠正候选人的错误；\n- 如果你认为用户连续几次回答的都不对，就少问一点；\n- 问完最后一个问题后，你可以问这样一个问题：上一份工作为什么离职？用户回答该问题后，请表示理解与支持。\n",
-                    'prompt_variables': [],
-                    'completion_params': {
-                        'max_token': 300,
-                        'temperature': 0.8,
-                        'top_p': 0.9,
-                        'presence_penalty': 0.1,
-                        'frequency_penalty': 0.1,
-                    }
-                },
-                opening_statement='你好，欢迎来参加我们的面试，我是这家科技公司的面试官，我将考察你的 Web 前端开发技能。接下来我会向您提出一些技术问题，请您尽可能详尽地回答。',
-                suggested_questions=None,
-                pre_prompt="你将扮演一个科技公司的面试官，考察用户作为候选人的 Web 前端开发水平，提出 5-10 个犀利的技术问题。\n\n请注意：\n- 每次只问一个问题\n- 用户回答问题后请直接问下一个问题，而不要试图纠正候选人的错误；\n- 如果你认为用户连续几次回答的都不对，就少问一点；\n- 问完最后一个问题后，你可以问这样一个问题：上一份工作为什么离职？用户回答该问题后，请表示理解与支持。\n",
-                model=json.dumps({
-                    "provider": "openai",
-                    "name": "gpt-3.5-turbo",
-                    "mode": "chat",
-                    "completion_params": {
-                        "max_tokens": 300,
-                        "temperature": 0.8,
-                        "top_p": 0.9,
-                        "presence_penalty": 0.1,
-                        "frequency_penalty": 0.1
-                    }
-                }),
-                user_input_form=None
-            )
+        'model_config': {
+            'model': {
+                "provider": "openai",
+                "name": "gpt-4",
+                "mode": "chat",
+                "completion_params": {}
+            }
         }
-    ],
+    }
 }

api/controllers/console/__init__.py

@@ -5,18 +5,18 @@ bp = Blueprint('console', __name__, url_prefix='/console/api')
 api = ExternalApi(bp)
 
 # Import other controllers
-from . import admin, apikey, extension, feature, setup, version
+from . import admin, apikey, extension, feature, setup, version, ping
 # Import app controllers
 from .app import (advanced_prompt_template, annotation, app, audio, completion, conversation, generator, message,
-                  model_config, site, statistic)
+                  model_config, site, statistic, workflow, workflow_run, workflow_app_log, workflow_statistic, agent)
 # Import auth controllers
 from .auth import activate, data_source_oauth, login, oauth
+# Import billing controllers
 from .billing import billing
 # Import datasets controllers
 from .datasets import data_source, datasets, datasets_document, datasets_segments, file, hit_testing
 # Import explore controllers
-from .explore import audio, completion, conversation, installed_app, message, parameter, recommended_app, saved_message
-# Import universal chat controllers
-from .universal_chat import audio, chat, conversation, message, parameter
+from .explore import (audio, completion, conversation, installed_app, message, parameter, recommended_app,
+                      saved_message, workflow)
 # Import workspace controllers
-from .workspace import account, members, model_providers, models, tool_providers, workspace
+from .workspace import account, members, model_providers, models, tool_providers, workspace

api/controllers/console/admin.py

@@ -1,14 +1,15 @@
 import os
 from functools import wraps
 
+from flask import request
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import NotFound, Unauthorized
+
+from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.wraps import only_edition_cloud
 from extensions.ext_database import db
-from flask import request
-from flask_restful import Resource, reqparse
-from libs.helper import supported_language
 from models.model import App, InstalledApp, RecommendedApp
-from werkzeug.exceptions import NotFound, Unauthorized
 
 
 def admin_required(view):

api/controllers/console/apikey.py

@@ -1,12 +1,13 @@
 import flask_restful
-from extensions.ext_database import db
 from flask_login import current_user
 from flask_restful import Resource, fields, marshal_with
+from werkzeug.exceptions import Forbidden
+
+from extensions.ext_database import db
 from libs.helper import TimestampField
 from libs.login import login_required
 from models.dataset import Dataset
 from models.model import ApiToken, App
-from werkzeug.exceptions import Forbidden
 
 from . import api
 from .setup import setup_required
@@ -61,9 +62,7 @@ class BaseApiKeyListResource(Resource):
         resource_id = str(resource_id)
         _get_resource(resource_id, current_user.current_tenant_id,
                       self.resource_model)
-
-        # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         current_key_count = db.session.query(ApiToken). \
@@ -102,7 +101,7 @@ class BaseApiKeyResource(Resource):
                       self.resource_model)
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         key = db.session.query(ApiToken). \

api/controllers/console/app/__init__.py

@@ -1,21 +0,0 @@
-from controllers.console.app.error import AppUnavailableError
-from extensions.ext_database import db
-from flask_login import current_user
-from models.model import App
-from werkzeug.exceptions import NotFound
-
-
-def _get_app(app_id, mode=None):
-    app = db.session.query(App).filter(
-        App.id == app_id,
-        App.tenant_id == current_user.current_tenant_id,
-        App.status == 'normal'
-    ).first()
-
-    if not app:
-        raise NotFound("App not found")
-
-    if mode and app.mode != mode:
-        raise NotFound("The {} app not found".format(mode))
-
-    return app

api/controllers/console/app/advanced_prompt_template.py

@@ -1,7 +1,8 @@
+from flask_restful import Resource, reqparse
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 from services.advanced_prompt_template_service import AdvancedPromptTemplateService
 

api/controllers/console/app/agent.py

@@ -0,0 +1,32 @@
+from flask_restful import Resource, reqparse
+
+from controllers.console import api
+from controllers.console.app.wraps import get_app_model
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from libs.helper import uuid_value
+from libs.login import login_required
+from models.model import AppMode
+from services.agent_service import AgentService
+
+
+class AgentLogApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.AGENT_CHAT])
+    def get(self, app_model):
+        """Get agent logs"""
+        parser = reqparse.RequestParser()
+        parser.add_argument('message_id', type=uuid_value, required=True, location='args')
+        parser.add_argument('conversation_id', type=uuid_value, required=True, location='args')
+
+        args = parser.parse_args()
+
+        return AgentService.get_agent_logs(
+            app_model,
+            args['conversation_id'],
+            args['message_id']
+        )
+    
+api.add_resource(AgentLogApi, '/apps/<uuid:app_id>/agent/logs')

api/controllers/console/app/annotation.py

@@ -1,17 +1,20 @@
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, marshal, marshal_with, reqparse
+from werkzeug.exceptions import Forbidden
+
 from controllers.console import api
 from controllers.console.app.error import NoFileUploadedError
 from controllers.console.datasets.error import TooManyFilesError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_redis import redis_client
-from fields.annotation_fields import (annotation_fields, annotation_hit_history_fields,
-                                      annotation_hit_history_list_fields, annotation_list_fields)
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal, marshal_with, reqparse
+from fields.annotation_fields import (
+    annotation_fields,
+    annotation_hit_history_fields,
+)
 from libs.login import login_required
 from services.annotation_service import AppAnnotationService
-from werkzeug.exceptions import Forbidden
 
 
 class AnnotationReplyActionApi(Resource):
@@ -21,7 +24,7 @@ class AnnotationReplyActionApi(Resource):
     @cloud_edition_billing_resource_check('annotation')
     def post(self, app_id, action):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)
@@ -45,7 +48,7 @@ class AppAnnotationSettingDetailApi(Resource):
     @account_initialization_required
     def get(self, app_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)
@@ -59,7 +62,7 @@ class AppAnnotationSettingUpdateApi(Resource):
     @account_initialization_required
     def post(self, app_id, annotation_setting_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)
@@ -80,7 +83,7 @@ class AnnotationReplyActionStatusApi(Resource):
     @cloud_edition_billing_resource_check('annotation')
     def get(self, app_id, job_id, action):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         job_id = str(job_id)
@@ -108,7 +111,7 @@ class AnnotationListApi(Resource):
     @account_initialization_required
     def get(self, app_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         page = request.args.get('page', default=1, type=int)
@@ -133,7 +136,7 @@ class AnnotationExportApi(Resource):
     @account_initialization_required
     def get(self, app_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)
@@ -152,7 +155,7 @@ class AnnotationCreateApi(Resource):
     @marshal_with(annotation_fields)
     def post(self, app_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)
@@ -172,7 +175,7 @@ class AnnotationUpdateDeleteApi(Resource):
     @marshal_with(annotation_fields)
     def post(self, app_id, annotation_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)
@@ -189,7 +192,7 @@ class AnnotationUpdateDeleteApi(Resource):
     @account_initialization_required
     def delete(self, app_id, annotation_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)
@@ -205,7 +208,7 @@ class AnnotationBatchImportApi(Resource):
     @cloud_edition_billing_resource_check('annotation')
     def post(self, app_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)
@@ -230,7 +233,7 @@ class AnnotationBatchImportStatusApi(Resource):
     @cloud_edition_billing_resource_check('annotation')
     def get(self, app_id, job_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         job_id = str(job_id)
@@ -257,7 +260,7 @@ class AnnotationHitHistoryListApi(Resource):
     @account_initialization_required
     def get(self, app_id, annotation_id):
         # The role of the current user in the table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         page = request.args.get('page', default=1, type=int)

api/controllers/console/app/app.py

@@ -1,34 +1,28 @@
-# -*- coding:utf-8 -*-
 import json
-import logging
-from datetime import datetime
 
-from constants.model_template import demo_model_templates, model_templates
+from flask_login import current_user
+from flask_restful import Resource, inputs, marshal_with, reqparse
+from werkzeug.exceptions import Forbidden, BadRequest
+
 from controllers.console import api
-from controllers.console.app.error import AppNotFoundError, ProviderNotInitializeError
+from controllers.console.app.wraps import get_app_model
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
-from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
-from core.model_manager import ModelManager
-from core.model_runtime.entities.model_entities import ModelType
-from core.provider_manager import ProviderManager
-from events.app_event import app_was_created, app_was_deleted
+from core.agent.entities import AgentToolEntity
 from extensions.ext_database import db
-from fields.app_fields import (app_detail_fields, app_detail_fields_with_site, app_pagination_fields,
-                               template_list_fields)
-from flask_login import current_user
-from flask_restful import Resource, abort, inputs, marshal_with, reqparse
+from fields.app_fields import (
+    app_detail_fields,
+    app_detail_fields_with_site,
+    app_pagination_fields,
+)
 from libs.login import login_required
-from models.model import App, AppModelConfig, Site
-from services.app_model_config_service import AppModelConfigService
-from werkzeug.exceptions import Forbidden
+from services.app_service import AppService
+from models.model import App, AppModelConfig, AppMode
+from core.tools.utils.configuration import ToolParameterConfigurationManager
+from core.tools.tool_manager import ToolManager
 
 
-def _get_app(app_id, tenant_id):
-    app = db.session.query(App).filter(App.id == app_id, App.tenant_id == tenant_id).first()
-    if not app:
-        raise AppNotFoundError
-    return app
+ALLOW_CREATE_APP_MODES = ['chat', 'agent-chat', 'advanced-chat', 'workflow', 'completion']
 
 
 class AppListApi(Resource):
@@ -42,16 +36,15 @@ class AppListApi(Resource):
         parser = reqparse.RequestParser()
         parser.add_argument('page', type=inputs.int_range(1, 99999), required=False, default=1, location='args')
         parser.add_argument('limit', type=inputs.int_range(1, 100), required=False, default=20, location='args')
+        parser.add_argument('mode', type=str, choices=['chat', 'workflow', 'agent-chat', 'channel', 'all'], default='all', location='args', required=False)
+        parser.add_argument('name', type=str, location='args', required=False)
         args = parser.parse_args()
 
-        app_models = db.paginate(
-            db.select(App).where(App.tenant_id == current_user.current_tenant_id,
-                                 App.is_universal == False).order_by(App.created_at.desc()),
-            page=args['page'],
-            per_page=args['limit'],
-            error_out=False)
+        # get app list
+        app_service = AppService()
+        app_pagination = app_service.get_paginate_apps(current_user.current_tenant_id, args)
 
-        return app_models
+        return app_pagination
 
     @setup_required
     @login_required
@@ -62,140 +55,49 @@ class AppListApi(Resource):
         """Create app"""
         parser = reqparse.RequestParser()
         parser.add_argument('name', type=str, required=True, location='json')
-        parser.add_argument('mode', type=str, choices=['completion', 'chat'], location='json')
+        parser.add_argument('description', type=str, location='json')
+        parser.add_argument('mode', type=str, choices=ALLOW_CREATE_APP_MODES, location='json')
         parser.add_argument('icon', type=str, location='json')
         parser.add_argument('icon_background', type=str, location='json')
-        parser.add_argument('model_config', type=dict, location='json')
         args = parser.parse_args()
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
-        try:
-            provider_manager = ProviderManager()
-            default_model_entity = provider_manager.get_default_model(
-                tenant_id=current_user.current_tenant_id,
-                model_type=ModelType.LLM
-            )
-        except (ProviderTokenNotInitError, LLMBadRequestError):
-            default_model_entity = None
-        except Exception as e:
-            logging.exception(e)
-            default_model_entity = None
+        if 'mode' not in args or args['mode'] is None:
+            raise BadRequest("mode is required")
 
-        if args['model_config'] is not None:
-            # validate config
-            model_config_dict = args['model_config']
-
-            # get model provider
-            model_manager = ModelManager()
-            model_instance = model_manager.get_default_model_instance(
-                tenant_id=current_user.current_tenant_id,
-                model_type=ModelType.LLM
-            )
-
-            if not model_instance:
-                raise ProviderNotInitializeError(
-                    f"No Default System Reasoning Model available. Please configure "
-                    f"in the Settings -> Model Provider.")
-            else:
-                model_config_dict["model"]["provider"] = model_instance.provider
-                model_config_dict["model"]["name"] = model_instance.model
-
-            model_configuration = AppModelConfigService.validate_configuration(
-                tenant_id=current_user.current_tenant_id,
-                account=current_user,
-                config=model_config_dict,
-                app_mode=args['mode']
-            )
-
-            app = App(
-                enable_site=True,
-                enable_api=True,
-                is_demo=False,
-                api_rpm=0,
-                api_rph=0,
-                status='normal'
-            )
-
-            app_model_config = AppModelConfig()
-            app_model_config = app_model_config.from_model_config_dict(model_configuration)
-        else:
-            if 'mode' not in args or args['mode'] is None:
-                abort(400, message="mode is required")
-
-            model_config_template = model_templates[args['mode'] + '_default']
-
-            app = App(**model_config_template['app'])
-            app_model_config = AppModelConfig(**model_config_template['model_config'])
-
-            # get model provider
-            model_manager = ModelManager()
-
-            try:
-                model_instance = model_manager.get_default_model_instance(
-                    tenant_id=current_user.current_tenant_id,
-                    model_type=ModelType.LLM
-                )
-            except ProviderTokenNotInitError:
-                model_instance = None
-
-            if model_instance:
-                model_dict = app_model_config.model_dict
-                model_dict['provider'] = model_instance.provider
-                model_dict['name'] = model_instance.model
-                app_model_config.model = json.dumps(model_dict)
-
-        app.name = args['name']
-        app.mode = args['mode']
-        app.icon = args['icon']
-        app.icon_background = args['icon_background']
-        app.tenant_id = current_user.current_tenant_id
-
-        db.session.add(app)
-        db.session.flush()
-
-        app_model_config.app_id = app.id
-        db.session.add(app_model_config)
-        db.session.flush()
-
-        app.app_model_config_id = app_model_config.id
-
-        account = current_user
-
-        site = Site(
-            app_id=app.id,
-            title=app.name,
-            default_language=account.interface_language,
-            customize_token_strategy='not_allow',
-            code=Site.generate_code(16)
-        )
-
-        db.session.add(site)
-        db.session.commit()
-
-        app_was_created.send(app)
+        app_service = AppService()
+        app = app_service.create_app(current_user.current_tenant_id, args, current_user)
 
         return app, 201
 
 
-class AppTemplateApi(Resource):
-
+class AppImportApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @marshal_with(template_list_fields)
-    def get(self):
-        """Get app demo templates"""
-        account = current_user
-        interface_language = account.interface_language
+    @marshal_with(app_detail_fields_with_site)
+    @cloud_edition_billing_resource_check('apps')
+    def post(self):
+        """Import app"""
+        # The role of the current user in the ta table must be admin or owner
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
 
-        templates = demo_model_templates.get(interface_language)
-        if not templates:
-            templates = demo_model_templates.get('en-US')
+        parser = reqparse.RequestParser()
+        parser.add_argument('data', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('name', type=str, location='json')
+        parser.add_argument('description', type=str, location='json')
+        parser.add_argument('icon', type=str, location='json')
+        parser.add_argument('icon_background', type=str, location='json')
+        args = parser.parse_args()
 
-        return {'data': templates}
+        app_service = AppService()
+        app = app_service.import_app(current_user.current_tenant_id, args['data'], args, current_user)
+
+        return app, 201
 
 
 class AppApi(Resource):
@@ -203,176 +105,198 @@ class AppApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
+    @get_app_model
     @marshal_with(app_detail_fields_with_site)
-    def get(self, app_id):
+    def get(self, app_model):
         """Get app detail"""
-        app_id = str(app_id)
-        app = _get_app(app_id, current_user.current_tenant_id)
+        # get original app model config
+        if app_model.mode == AppMode.AGENT_CHAT.value or app_model.is_agent:
+            model_config: AppModelConfig = app_model.app_model_config
+            agent_mode = model_config.agent_mode_dict
+            # decrypt agent tool parameters if it's secret-input
+            for tool in agent_mode.get('tools') or []:
+                if not isinstance(tool, dict) or len(tool.keys()) <= 3:
+                    continue
+                agent_tool_entity = AgentToolEntity(**tool)
+                # get tool
+                try:
+                    tool_runtime = ToolManager.get_agent_tool_runtime(
+                        tenant_id=current_user.current_tenant_id,
+                        agent_tool=agent_tool_entity,
+                    )
+                    manager = ToolParameterConfigurationManager(
+                        tenant_id=current_user.current_tenant_id,
+                        tool_runtime=tool_runtime,
+                        provider_name=agent_tool_entity.provider_id,
+                        provider_type=agent_tool_entity.provider_type,
+                    )
 
-        return app
+                    # get decrypted parameters
+                    if agent_tool_entity.tool_parameters:
+                        parameters = manager.decrypt_tool_parameters(agent_tool_entity.tool_parameters or {})
+                        masked_parameter = manager.mask_tool_parameters(parameters or {})
+                    else:
+                        masked_parameter = {}
+
+                    # override tool parameters
+                    tool['tool_parameters'] = masked_parameter
+                except Exception as e:
+                    pass
+
+            # override agent mode
+            model_config.agent_mode = json.dumps(agent_mode)
+            db.session.commit()
+
+        return app_model
 
     @setup_required
     @login_required
     @account_initialization_required
-    def delete(self, app_id):
-        """Delete app"""
-        app_id = str(app_id)
+    @get_app_model
+    @marshal_with(app_detail_fields_with_site)
+    def put(self, app_model):
+        """Update app"""
+        parser = reqparse.RequestParser()
+        parser.add_argument('name', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('description', type=str, location='json')
+        parser.add_argument('icon', type=str, location='json')
+        parser.add_argument('icon_background', type=str, location='json')
+        args = parser.parse_args()
 
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        app_service = AppService()
+        app_model = app_service.update_app(app_model, args)
+
+        return app_model
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model
+    def delete(self, app_model):
+        """Delete app"""
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
-        app = _get_app(app_id, current_user.current_tenant_id)
-
-        db.session.delete(app)
-        db.session.commit()
-
-        # todo delete related data??
-        # model_config, site, api_token, conversation, message, message_feedback, message_annotation
-
-        app_was_deleted.send(app)
+        app_service = AppService()
+        app_service.delete_app(app_model)
 
         return {'result': 'success'}, 204
 
 
+class AppCopyApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model
+    @marshal_with(app_detail_fields_with_site)
+    def post(self, app_model):
+        """Copy app"""
+        # The role of the current user in the ta table must be admin or owner
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('name', type=str, location='json')
+        parser.add_argument('description', type=str, location='json')
+        parser.add_argument('icon', type=str, location='json')
+        parser.add_argument('icon_background', type=str, location='json')
+        args = parser.parse_args()
+
+        app_service = AppService()
+        data = app_service.export_app(app_model)
+        app = app_service.import_app(current_user.current_tenant_id, data, args, current_user)
+
+        return app, 201
+
+
+class AppExportApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model
+    def get(self, app_model):
+        """Export app"""
+        app_service = AppService()
+
+        return {
+            "data": app_service.export_app(app_model)
+        }
+
+
 class AppNameApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
+    @get_app_model
     @marshal_with(app_detail_fields)
-    def post(self, app_id):
-        app_id = str(app_id)
-        app = _get_app(app_id, current_user.current_tenant_id)
-
+    def post(self, app_model):
         parser = reqparse.RequestParser()
         parser.add_argument('name', type=str, required=True, location='json')
         args = parser.parse_args()
 
-        app.name = args.get('name')
-        app.updated_at = datetime.utcnow()
-        db.session.commit()
-        return app
+        app_service = AppService()
+        app_model = app_service.update_app_name(app_model, args.get('name'))
+
+        return app_model
 
 
 class AppIconApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
+    @get_app_model
     @marshal_with(app_detail_fields)
-    def post(self, app_id):
-        app_id = str(app_id)
-        app = _get_app(app_id, current_user.current_tenant_id)
-
+    def post(self, app_model):
         parser = reqparse.RequestParser()
         parser.add_argument('icon', type=str, location='json')
         parser.add_argument('icon_background', type=str, location='json')
         args = parser.parse_args()
 
-        app.icon = args.get('icon')
-        app.icon_background = args.get('icon_background')
-        app.updated_at = datetime.utcnow()
-        db.session.commit()
+        app_service = AppService()
+        app_model = app_service.update_app_icon(app_model, args.get('icon'), args.get('icon_background'))
 
-        return app
+        return app_model
 
 
 class AppSiteStatus(Resource):
     @setup_required
     @login_required
     @account_initialization_required
+    @get_app_model
     @marshal_with(app_detail_fields)
-    def post(self, app_id):
+    def post(self, app_model):
         parser = reqparse.RequestParser()
         parser.add_argument('enable_site', type=bool, required=True, location='json')
         args = parser.parse_args()
-        app_id = str(app_id)
-        app = db.session.query(App).filter(App.id == app_id, App.tenant_id == current_user.current_tenant_id).first()
-        if not app:
-            raise AppNotFoundError
 
-        if args.get('enable_site') == app.enable_site:
-            return app
+        app_service = AppService()
+        app_model = app_service.update_app_site_status(app_model, args.get('enable_site'))
 
-        app.enable_site = args.get('enable_site')
-        app.updated_at = datetime.utcnow()
-        db.session.commit()
-        return app
+        return app_model
 
 
 class AppApiStatus(Resource):
     @setup_required
     @login_required
     @account_initialization_required
+    @get_app_model
     @marshal_with(app_detail_fields)
-    def post(self, app_id):
+    def post(self, app_model):
         parser = reqparse.RequestParser()
         parser.add_argument('enable_api', type=bool, required=True, location='json')
         args = parser.parse_args()
 
-        app_id = str(app_id)
-        app = _get_app(app_id, current_user.current_tenant_id)
+        app_service = AppService()
+        app_model = app_service.update_app_api_status(app_model, args.get('enable_api'))
 
-        if args.get('enable_api') == app.enable_api:
-            return app
-
-        app.enable_api = args.get('enable_api')
-        app.updated_at = datetime.utcnow()
-        db.session.commit()
-        return app
-
-
-class AppCopy(Resource):
-    @staticmethod
-    def create_app_copy(app):
-        copy_app = App(
-            name=app.name + ' copy',
-            icon=app.icon,
-            icon_background=app.icon_background,
-            tenant_id=app.tenant_id,
-            mode=app.mode,
-            app_model_config_id=app.app_model_config_id,
-            enable_site=app.enable_site,
-            enable_api=app.enable_api,
-            api_rpm=app.api_rpm,
-            api_rph=app.api_rph
-        )
-        return copy_app
-
-    @staticmethod
-    def create_app_model_config_copy(app_config, copy_app_id):
-        copy_app_model_config = app_config.copy()
-        copy_app_model_config.app_id = copy_app_id
-
-        return copy_app_model_config
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @marshal_with(app_detail_fields)
-    def post(self, app_id):
-        app_id = str(app_id)
-        app = _get_app(app_id, current_user.current_tenant_id)
-
-        copy_app = self.create_app_copy(app)
-        db.session.add(copy_app)
-
-        app_config = db.session.query(AppModelConfig). \
-            filter(AppModelConfig.app_id == app_id). \
-            one_or_none()
-
-        if app_config:
-            copy_app_model_config = self.create_app_model_config_copy(app_config, copy_app.id)
-            db.session.add(copy_app_model_config)
-            db.session.commit()
-            copy_app.app_model_config_id = copy_app_model_config.id
-        db.session.commit()
-
-        return copy_app, 201
+        return app_model
 
 
 api.add_resource(AppListApi, '/apps')
-api.add_resource(AppTemplateApi, '/app-templates')
+api.add_resource(AppImportApi, '/apps/import')
 api.add_resource(AppApi, '/apps/<uuid:app_id>')
-api.add_resource(AppCopy, '/apps/<uuid:app_id>/copy')
+api.add_resource(AppCopyApi, '/apps/<uuid:app_id>/copy')
+api.add_resource(AppExportApi, '/apps/<uuid:app_id>/export')
 api.add_resource(AppNameApi, '/apps/<uuid:app_id>/name')
 api.add_resource(AppIconApi, '/apps/<uuid:app_id>/icon')
 api.add_resource(AppSiteStatus, '/apps/<uuid:app_id>/site-enable')

api/controllers/console/app/audio.py

@@ -1,40 +1,51 @@
-# -*- coding:utf-8 -*-
 import logging
 
+from flask import request
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import InternalServerError
+
 import services
 from controllers.console import api
-from controllers.console.app import _get_app
-from controllers.console.app.error import (AppUnavailableError, AudioTooLargeError, CompletionRequestError,
-                                           NoAudioUploadedError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderNotSupportSpeechToTextError,
-                                           ProviderQuotaExceededError, UnsupportedAudioTypeError)
+from controllers.console.app.error import (
+    AppUnavailableError,
+    AudioTooLargeError,
+    CompletionRequestError,
+    NoAudioUploadedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderNotSupportSpeechToTextError,
+    ProviderQuotaExceededError,
+    UnsupportedAudioTypeError,
+)
+from controllers.console.app.wraps import get_app_model
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import request
-from flask_restful import Resource
 from libs.login import login_required
+from models.model import AppMode
 from services.audio_service import AudioService
-from services.errors.audio import (AudioTooLargeServiceError, NoAudioUploadedServiceError,
-                                   ProviderNotSupportSpeechToTextServiceError, UnsupportedAudioTypeServiceError)
-from werkzeug.exceptions import InternalServerError
+from services.errors.audio import (
+    AudioTooLargeServiceError,
+    NoAudioUploadedServiceError,
+    ProviderNotSupportSpeechToTextServiceError,
+    UnsupportedAudioTypeServiceError,
+)
 
 
 class ChatMessageAudioApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def post(self, app_id):
-        app_id = str(app_id)
-        app_model = _get_app(app_id, 'chat')
-
+    @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
+    def post(self, app_model):
         file = request.files['file']
 
         try:
-            response = AudioService.transcript(
-                tenant_id=app_model.tenant_id,
+            response = AudioService.transcript_asr(
+                app_model=app_model,
                 file=file,
+                end_user=None,
             )
 
             return response
@@ -60,8 +71,93 @@ class ChatMessageAudioApi(Resource):
         except ValueError as e:
             raise e
         except Exception as e:
-            logging.exception("internal server error.")
+            logging.exception(f"internal server error, {str(e)}.")
             raise InternalServerError()
-        
 
-api.add_resource(ChatMessageAudioApi, '/apps/<uuid:app_id>/audio-to-text')
+
+class ChatMessageTextApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model
+    def post(self, app_model):
+        try:
+            response = AudioService.transcript_tts(
+                app_model=app_model,
+                text=request.form['text'],
+                voice=request.form.get('voice'),
+                streaming=False
+            )
+
+            return {'data': response.data.decode('latin1')}
+        except services.errors.app_model_config.AppModelConfigBrokenError:
+            logging.exception("App model config broken.")
+            raise AppUnavailableError()
+        except NoAudioUploadedServiceError:
+            raise NoAudioUploadedError()
+        except AudioTooLargeServiceError as e:
+            raise AudioTooLargeError(str(e))
+        except UnsupportedAudioTypeServiceError:
+            raise UnsupportedAudioTypeError()
+        except ProviderNotSupportSpeechToTextServiceError:
+            raise ProviderNotSupportSpeechToTextError()
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except InvokeError as e:
+            raise CompletionRequestError(e.description)
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logging.exception(f"internal server error, {str(e)}.")
+            raise InternalServerError()
+
+
+class TextModesApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model
+    def get(self, app_model):
+        try:
+            parser = reqparse.RequestParser()
+            parser.add_argument('language', type=str, required=True, location='args')
+            args = parser.parse_args()
+
+            response = AudioService.transcript_tts_voices(
+                tenant_id=app_model.tenant_id,
+                language=args['language'],
+            )
+
+            return response
+        except services.errors.audio.ProviderNotSupportTextToSpeechLanageServiceError:
+            raise AppUnavailableError("Text to audio voices language parameter loss.")
+        except NoAudioUploadedServiceError:
+            raise NoAudioUploadedError()
+        except AudioTooLargeServiceError as e:
+            raise AudioTooLargeError(str(e))
+        except UnsupportedAudioTypeServiceError:
+            raise UnsupportedAudioTypeError()
+        except ProviderNotSupportSpeechToTextServiceError:
+            raise ProviderNotSupportSpeechToTextError()
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except InvokeError as e:
+            raise CompletionRequestError(e.description)
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logging.exception(f"internal server error, {str(e)}.")
+            raise InternalServerError()
+
+
+api.add_resource(ChatMessageAudioApi, '/apps/<uuid:app_id>/audio-to-text')
+api.add_resource(ChatMessageTextApi, '/apps/<uuid:app_id>/text-to-audio')
+api.add_resource(TextModesApi, '/apps/<uuid:app_id>/text-to-audio/voices')

api/controllers/console/app/completion.py

@@ -1,27 +1,31 @@
-# -*- coding:utf-8 -*-
-import json
 import logging
-from typing import Generator, Union
 
 import flask_login
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import InternalServerError, NotFound
+
 import services
 from controllers.console import api
-from controllers.console.app import _get_app
-from controllers.console.app.error import (AppUnavailableError, CompletionRequestError, ConversationCompletedError,
-                                           ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    AppUnavailableError,
+    CompletionRequestError,
+    ConversationCompletedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
+from controllers.console.app.wraps import get_app_model
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
-from core.application_queue_manager import ApplicationQueueManager
-from core.entities.application_entities import InvokeFrom
+from core.app.apps.base_app_queue_manager import AppQueueManager
+from core.app.entities.app_invoke_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import Response, stream_with_context
-from flask_restful import Resource, reqparse
+from libs import helper
 from libs.helper import uuid_value
 from libs.login import login_required
-from services.completion_service import CompletionService
-from werkzeug.exceptions import InternalServerError, NotFound
+from models.model import AppMode
+from services.app_generate_service import AppGenerateService
 
 
 # define completion message api for user
@@ -30,12 +34,8 @@ class CompletionMessageApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def post(self, app_id):
-        app_id = str(app_id)
-
-        # get app info
-        app_model = _get_app(app_id, 'completion')
-
+    @get_app_model(mode=AppMode.COMPLETION)
+    def post(self, app_model):
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, required=True, location='json')
         parser.add_argument('query', type=str, location='json', default='')
@@ -51,16 +51,15 @@ class CompletionMessageApi(Resource):
         account = flask_login.current_user
 
         try:
-            response = CompletionService.completion(
+            response = AppGenerateService.generate(
                 app_model=app_model,
                 user=account,
                 args=args,
                 invoke_from=InvokeFrom.DEBUGGER,
-                streaming=streaming,
-                is_model_config_override=True
+                streaming=streaming
             )
 
-            return compact_response(response)
+            return helper.compact_generate_response(response)
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
         except services.errors.conversation.ConversationCompletedError:
@@ -87,15 +86,11 @@ class CompletionMessageStopApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def post(self, app_id, task_id):
-        app_id = str(app_id)
-
-        # get app info
-        _get_app(app_id, 'completion')
-
+    @get_app_model(mode=AppMode.COMPLETION)
+    def post(self, app_model, task_id):
         account = flask_login.current_user
 
-        ApplicationQueueManager.set_stop_flag(task_id, InvokeFrom.DEBUGGER, account.id)
+        AppQueueManager.set_stop_flag(task_id, InvokeFrom.DEBUGGER, account.id)
 
         return {'result': 'success'}, 200
 
@@ -104,12 +99,8 @@ class ChatMessageApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def post(self, app_id):
-        app_id = str(app_id)
-
-        # get app info
-        app_model = _get_app(app_id, 'chat')
-
+    @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT])
+    def post(self, app_model):
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, required=True, location='json')
         parser.add_argument('query', type=str, required=True, location='json')
@@ -126,16 +117,15 @@ class ChatMessageApi(Resource):
         account = flask_login.current_user
 
         try:
-            response = CompletionService.completion(
+            response = AppGenerateService.generate(
                 app_model=app_model,
                 user=account,
                 args=args,
                 invoke_from=InvokeFrom.DEBUGGER,
-                streaming=streaming,
-                is_model_config_override=True
+                streaming=streaming
             )
 
-            return compact_response(response)
+            return helper.compact_generate_response(response)
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
         except services.errors.conversation.ConversationCompletedError:
@@ -158,52 +148,15 @@ class ChatMessageApi(Resource):
             raise InternalServerError()
 
 
-def compact_response(response: Union[dict, Generator]) -> Response:
-    if isinstance(response, dict):
-        return Response(response=json.dumps(response), status=200, mimetype='application/json')
-    else:
-        def generate() -> Generator:
-            try:
-                for chunk in response:
-                    yield chunk
-            except services.errors.conversation.ConversationNotExistsError:
-                yield "data: " + json.dumps(api.handle_error(NotFound("Conversation Not Exists.")).get_json()) + "\n\n"
-            except services.errors.conversation.ConversationCompletedError:
-                yield "data: " + json.dumps(api.handle_error(ConversationCompletedError()).get_json()) + "\n\n"
-            except services.errors.app_model_config.AppModelConfigBrokenError:
-                logging.exception("App model config broken.")
-                yield "data: " + json.dumps(api.handle_error(AppUnavailableError()).get_json()) + "\n\n"
-            except ProviderTokenNotInitError as ex:
-                yield "data: " + json.dumps(api.handle_error(ProviderNotInitializeError(ex.description)).get_json()) + "\n\n"
-            except QuotaExceededError:
-                yield "data: " + json.dumps(api.handle_error(ProviderQuotaExceededError()).get_json()) + "\n\n"
-            except ModelCurrentlyNotSupportError:
-                yield "data: " + json.dumps(api.handle_error(ProviderModelCurrentlyNotSupportError()).get_json()) + "\n\n"
-            except InvokeError as e:
-                yield "data: " + json.dumps(api.handle_error(CompletionRequestError(e.description)).get_json()) + "\n\n"
-            except ValueError as e:
-                yield "data: " + json.dumps(api.handle_error(e).get_json()) + "\n\n"
-            except Exception:
-                logging.exception("internal server error.")
-                yield "data: " + json.dumps(api.handle_error(InternalServerError()).get_json()) + "\n\n"
-
-        return Response(stream_with_context(generate()), status=200,
-                        mimetype='text/event-stream')
-
-
 class ChatMessageStopApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def post(self, app_id, task_id):
-        app_id = str(app_id)
-
-        # get app info
-        _get_app(app_id, 'chat')
-
+    @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
+    def post(self, app_model, task_id):
         account = flask_login.current_user
 
-        ApplicationQueueManager.set_stop_flag(task_id, InvokeFrom.DEBUGGER, account.id)
+        AppQueueManager.set_stop_flag(task_id, InvokeFrom.DEBUGGER, account.id)
 
         return {'result': 'success'}, 200
 

api/controllers/console/app/conversation.py

@@ -1,33 +1,37 @@
 from datetime import datetime
 
 import pytz
-from controllers.console import api
-from controllers.console.app import _get_app
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
-from extensions.ext_database import db
-from fields.conversation_fields import (conversation_detail_fields, conversation_message_detail_fields,
-                                        conversation_pagination_fields, conversation_with_summary_pagination_fields)
 from flask_login import current_user
 from flask_restful import Resource, marshal_with, reqparse
 from flask_restful.inputs import int_range
-from libs.helper import datetime_string
-from libs.login import login_required
-from models.model import Conversation, Message, MessageAnnotation
 from sqlalchemy import func, or_
 from sqlalchemy.orm import joinedload
 from werkzeug.exceptions import NotFound
 
+from controllers.console import api
+from controllers.console.app.wraps import get_app_model
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from extensions.ext_database import db
+from fields.conversation_fields import (
+    conversation_detail_fields,
+    conversation_message_detail_fields,
+    conversation_pagination_fields,
+    conversation_with_summary_pagination_fields,
+)
+from libs.helper import datetime_string
+from libs.login import login_required
+from models.model import AppMode, Conversation, Message, MessageAnnotation
+
 
 class CompletionConversationApi(Resource):
 
     @setup_required
     @login_required
     @account_initialization_required
+    @get_app_model(mode=AppMode.COMPLETION)
     @marshal_with(conversation_pagination_fields)
-    def get(self, app_id):
-        app_id = str(app_id)
-
+    def get(self, app_model):
         parser = reqparse.RequestParser()
         parser.add_argument('keyword', type=str, location='args')
         parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
@@ -38,10 +42,7 @@ class CompletionConversationApi(Resource):
         parser.add_argument('limit', type=int_range(1, 100), default=20, location='args')
         args = parser.parse_args()
 
-        # get app info
-        app = _get_app(app_id, 'completion')
-
-        query = db.select(Conversation).where(Conversation.app_id == app.id, Conversation.mode == 'completion')
+        query = db.select(Conversation).where(Conversation.app_id == app_model.id, Conversation.mode == 'completion')
 
         if args['keyword']:
             query = query.join(
@@ -101,24 +102,22 @@ class CompletionConversationDetailApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
+    @get_app_model(mode=AppMode.COMPLETION)
     @marshal_with(conversation_message_detail_fields)
-    def get(self, app_id, conversation_id):
-        app_id = str(app_id)
+    def get(self, app_model, conversation_id):
         conversation_id = str(conversation_id)
 
-        return _get_conversation(app_id, conversation_id, 'completion')
+        return _get_conversation(app_model, conversation_id)
 
     @setup_required
     @login_required
     @account_initialization_required
-    def delete(self, app_id, conversation_id):
-        app_id = str(app_id)
+    @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
+    def delete(self, app_model, conversation_id):
         conversation_id = str(conversation_id)
 
-        app = _get_app(app_id, 'chat')
-
         conversation = db.session.query(Conversation) \
-            .filter(Conversation.id == conversation_id, Conversation.app_id == app.id).first()
+            .filter(Conversation.id == conversation_id, Conversation.app_id == app_model.id).first()
 
         if not conversation:
             raise NotFound("Conversation Not Exists.")
@@ -134,10 +133,9 @@ class ChatConversationApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
+    @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
     @marshal_with(conversation_with_summary_pagination_fields)
-    def get(self, app_id):
-        app_id = str(app_id)
-
+    def get(self, app_model):
         parser = reqparse.RequestParser()
         parser.add_argument('keyword', type=str, location='args')
         parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
@@ -149,10 +147,7 @@ class ChatConversationApi(Resource):
         parser.add_argument('limit', type=int_range(1, 100), required=False, default=20, location='args')
         args = parser.parse_args()
 
-        # get app info
-        app = _get_app(app_id, 'chat')
-
-        query = db.select(Conversation).where(Conversation.app_id == app.id, Conversation.mode == 'chat')
+        query = db.select(Conversation).where(Conversation.app_id == app_model.id)
 
         if args['keyword']:
             query = query.join(
@@ -206,6 +201,9 @@ class ChatConversationApi(Resource):
                 .having(func.count(Message.id) >= args['message_count_gte'])
             )
 
+        if app_model.mode == AppMode.ADVANCED_CHAT.value:
+            query = query.where(Conversation.override_model_configs.is_(None))
+
         query = query.order_by(Conversation.created_at.desc())
 
         conversations = db.paginate(
@@ -223,25 +221,22 @@ class ChatConversationDetailApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
+    @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
     @marshal_with(conversation_detail_fields)
-    def get(self, app_id, conversation_id):
-        app_id = str(app_id)
+    def get(self, app_model, conversation_id):
         conversation_id = str(conversation_id)
 
-        return _get_conversation(app_id, conversation_id, 'chat')
+        return _get_conversation(app_model, conversation_id)
 
     @setup_required
     @login_required
+    @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
     @account_initialization_required
-    def delete(self, app_id, conversation_id):
-        app_id = str(app_id)
+    def delete(self, app_model, conversation_id):
         conversation_id = str(conversation_id)
 
-        # get app info
-        app = _get_app(app_id, 'chat')
-
         conversation = db.session.query(Conversation) \
-            .filter(Conversation.id == conversation_id, Conversation.app_id == app.id).first()
+            .filter(Conversation.id == conversation_id, Conversation.app_id == app_model.id).first()
 
         if not conversation:
             raise NotFound("Conversation Not Exists.")
@@ -258,12 +253,9 @@ api.add_resource(ChatConversationApi, '/apps/<uuid:app_id>/chat-conversations')
 api.add_resource(ChatConversationDetailApi, '/apps/<uuid:app_id>/chat-conversations/<uuid:conversation_id>')
 
 
-def _get_conversation(app_id, conversation_id, mode):
-    # get app info
-    app = _get_app(app_id, mode)
-
+def _get_conversation(app_model, conversation_id):
     conversation = db.session.query(Conversation) \
-        .filter(Conversation.id == conversation_id, Conversation.app_id == app.id).first()
+        .filter(Conversation.id == conversation_id, Conversation.app_id == app_model.id).first()
 
     if not conversation:
         raise NotFound("Conversation Not Exists.")

api/controllers/console/app/error.py

@@ -85,3 +85,9 @@ class TooManyFilesError(BaseHTTPException):
     error_code = 'too_many_files'
     description = "Only one file is allowed."
     code = 400
+
+
+class DraftWorkflowNotExist(BaseHTTPException):
+    error_code = 'draft_workflow_not_exist'
+    description = "Draft workflow need to be initialized."
+    code = 400

api/controllers/console/app/generator.py

@@ -1,13 +1,18 @@
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+
 from controllers.console import api
-from controllers.console.app.error import (CompletionRequestError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
-from core.generator.llm_generator import LLMGenerator
+from core.llm_generator.llm_generator import LLMGenerator
 from core.model_runtime.errors.invoke import InvokeError
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 
 

api/controllers/console/app/message.py

@@ -1,34 +1,34 @@
-import json
 import logging
-from typing import Generator, Union
+
+from flask_login import current_user
+from flask_restful import Resource, fields, marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 from controllers.console import api
-from controllers.console.app import _get_app
-from controllers.console.app.error import (AppMoreLikeThisDisabledError, CompletionRequestError,
-                                           ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
+from controllers.console.app.wraps import get_app_model
+from controllers.console.explore.error import AppSuggestedQuestionsAfterAnswerDisabledError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
-from core.entities.application_entities import InvokeFrom
+from core.app.entities.app_invoke_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
 from extensions.ext_database import db
 from fields.conversation_fields import annotation_fields, message_detail_fields
-from flask import Response, stream_with_context
-from flask_login import current_user
-from flask_restful import Resource, fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
 from libs.helper import uuid_value
 from libs.infinite_scroll_pagination import InfiniteScrollPagination
 from libs.login import login_required
-from models.model import Conversation, Message, MessageAnnotation, MessageFeedback
+from models.model import AppMode, Conversation, Message, MessageAnnotation, MessageFeedback
 from services.annotation_service import AppAnnotationService
-from services.completion_service import CompletionService
-from services.errors.app import MoreLikeThisDisabledError
 from services.errors.conversation import ConversationNotExistsError
-from services.errors.message import MessageNotExistsError
+from services.errors.message import MessageNotExistsError, SuggestedQuestionsAfterAnswerDisabledError
 from services.message_service import MessageService
-from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 
 class ChatMessageListApi(Resource):
@@ -40,14 +40,10 @@ class ChatMessageListApi(Resource):
 
     @setup_required
     @login_required
+    @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
     @account_initialization_required
     @marshal_with(message_infinite_scroll_pagination_fields)
-    def get(self, app_id):
-        app_id = str(app_id)
-
-        # get app info
-        app = _get_app(app_id, 'chat')
-
+    def get(self, app_model):
         parser = reqparse.RequestParser()
         parser.add_argument('conversation_id', required=True, type=uuid_value, location='args')
         parser.add_argument('first_id', type=uuid_value, location='args')
@@ -56,7 +52,7 @@ class ChatMessageListApi(Resource):
 
         conversation = db.session.query(Conversation).filter(
             Conversation.id == args['conversation_id'],
-            Conversation.app_id == app.id
+            Conversation.app_id == app_model.id
         ).first()
 
         if not conversation:
@@ -104,12 +100,8 @@ class MessageFeedbackApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def post(self, app_id):
-        app_id = str(app_id)
-
-        # get app info
-        app = _get_app(app_id)
-
+    @get_app_model
+    def post(self, app_model):
         parser = reqparse.RequestParser()
         parser.add_argument('message_id', required=True, type=uuid_value, location='json')
         parser.add_argument('rating', type=str, choices=['like', 'dislike', None], location='json')
@@ -119,7 +111,7 @@ class MessageFeedbackApi(Resource):
 
         message = db.session.query(Message).filter(
             Message.id == message_id,
-            Message.app_id == app.id
+            Message.app_id == app_model.id
         ).first()
 
         if not message:
@@ -135,7 +127,7 @@ class MessageFeedbackApi(Resource):
             raise ValueError('rating cannot be None when feedback not exists')
         else:
             feedback = MessageFeedback(
-                app_id=app.id,
+                app_id=app_model.id,
                 conversation_id=message.conversation_id,
                 message_id=message.id,
                 rating=args['rating'],
@@ -154,21 +146,20 @@ class MessageAnnotationApi(Resource):
     @login_required
     @account_initialization_required
     @cloud_edition_billing_resource_check('annotation')
+    @get_app_model
     @marshal_with(annotation_fields)
-    def post(self, app_id):
+    def post(self, app_model):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
-        app_id = str(app_id)
-
         parser = reqparse.RequestParser()
         parser.add_argument('message_id', required=False, type=uuid_value, location='json')
         parser.add_argument('question', required=True, type=str, location='json')
         parser.add_argument('answer', required=True, type=str, location='json')
         parser.add_argument('annotation_reply', required=False, type=dict, location='json')
         args = parser.parse_args()
-        annotation = AppAnnotationService.up_insert_app_annotation_from_message(args, app_id)
+        annotation = AppAnnotationService.up_insert_app_annotation_from_message(args, app_model.id)
 
         return annotation
 
@@ -177,113 +168,29 @@ class MessageAnnotationCountApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def get(self, app_id):
-        app_id = str(app_id)
-
-        # get app info
-        app = _get_app(app_id)
-
+    @get_app_model
+    def get(self, app_model):
         count = db.session.query(MessageAnnotation).filter(
-            MessageAnnotation.app_id == app.id
+            MessageAnnotation.app_id == app_model.id
         ).count()
 
         return {'count': count}
 
 
-class MessageMoreLikeThisApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def get(self, app_id, message_id):
-        app_id = str(app_id)
-        message_id = str(message_id)
-
-        parser = reqparse.RequestParser()
-        parser.add_argument('response_mode', type=str, required=True, choices=['blocking', 'streaming'],
-                            location='args')
-        args = parser.parse_args()
-
-        streaming = args['response_mode'] == 'streaming'
-
-        # get app info
-        app_model = _get_app(app_id, 'completion')
-
-        try:
-            response = CompletionService.generate_more_like_this(
-                app_model=app_model,
-                user=current_user,
-                message_id=message_id,
-                invoke_from=InvokeFrom.DEBUGGER,
-                streaming=streaming
-            )
-            return compact_response(response)
-        except MessageNotExistsError:
-            raise NotFound("Message Not Exists.")
-        except MoreLikeThisDisabledError:
-            raise AppMoreLikeThisDisabledError()
-        except ProviderTokenNotInitError as ex:
-            raise ProviderNotInitializeError(ex.description)
-        except QuotaExceededError:
-            raise ProviderQuotaExceededError()
-        except ModelCurrentlyNotSupportError:
-            raise ProviderModelCurrentlyNotSupportError()
-        except InvokeError as e:
-            raise CompletionRequestError(e.description)
-        except ValueError as e:
-            raise e
-        except Exception as e:
-            logging.exception("internal server error.")
-            raise InternalServerError()
-
-
-def compact_response(response: Union[dict, Generator]) -> Response:
-    if isinstance(response, dict):
-        return Response(response=json.dumps(response), status=200, mimetype='application/json')
-    else:
-        def generate() -> Generator:
-            try:
-                for chunk in response:
-                    yield chunk
-            except MessageNotExistsError:
-                yield "data: " + json.dumps(api.handle_error(NotFound("Message Not Exists.")).get_json()) + "\n\n"
-            except MoreLikeThisDisabledError:
-                yield "data: " + json.dumps(api.handle_error(AppMoreLikeThisDisabledError()).get_json()) + "\n\n"
-            except ProviderTokenNotInitError as ex:
-                yield "data: " + json.dumps(api.handle_error(ProviderNotInitializeError(ex.description)).get_json()) + "\n\n"
-            except QuotaExceededError:
-                yield "data: " + json.dumps(api.handle_error(ProviderQuotaExceededError()).get_json()) + "\n\n"
-            except ModelCurrentlyNotSupportError:
-                yield "data: " + json.dumps(
-                    api.handle_error(ProviderModelCurrentlyNotSupportError()).get_json()) + "\n\n"
-            except InvokeError as e:
-                yield "data: " + json.dumps(api.handle_error(CompletionRequestError(e.description)).get_json()) + "\n\n"
-            except ValueError as e:
-                yield "data: " + json.dumps(api.handle_error(e).get_json()) + "\n\n"
-            except Exception:
-                logging.exception("internal server error.")
-                yield "data: " + json.dumps(api.handle_error(InternalServerError()).get_json()) + "\n\n"
-
-        return Response(stream_with_context(generate()), status=200,
-                        mimetype='text/event-stream')
-
-
 class MessageSuggestedQuestionApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def get(self, app_id, message_id):
-        app_id = str(app_id)
+    @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
+    def get(self, app_model, message_id):
         message_id = str(message_id)
 
-        # get app info
-        app_model = _get_app(app_id, 'chat')
-
         try:
             questions = MessageService.get_suggested_questions_after_answer(
                 app_model=app_model,
                 message_id=message_id,
                 user=current_user,
-                check_enabled=False
+                invoke_from=InvokeFrom.DEBUGGER
             )
         except MessageNotExistsError:
             raise NotFound("Message not found")
@@ -297,6 +204,8 @@ class MessageSuggestedQuestionApi(Resource):
             raise ProviderModelCurrentlyNotSupportError()
         except InvokeError as e:
             raise CompletionRequestError(e.description)
+        except SuggestedQuestionsAfterAnswerDisabledError:
+            raise AppSuggestedQuestionsAfterAnswerDisabledError()
         except Exception:
             logging.exception("internal server error.")
             raise InternalServerError()
@@ -308,14 +217,11 @@ class MessageApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
+    @get_app_model
     @marshal_with(message_detail_fields)
-    def get(self, app_id, message_id):
-        app_id = str(app_id)
+    def get(self, app_model, message_id):
         message_id = str(message_id)
 
-        # get app info
-        app_model = _get_app(app_id)
-
         message = db.session.query(Message).filter(
             Message.id == message_id,
             Message.app_id == app_model.id
@@ -327,7 +233,6 @@ class MessageApi(Resource):
         return message
 
 
-api.add_resource(MessageMoreLikeThisApi, '/apps/<uuid:app_id>/completion-messages/<uuid:message_id>/more-like-this')
 api.add_resource(MessageSuggestedQuestionApi, '/apps/<uuid:app_id>/chat-messages/<uuid:message_id>/suggested-questions')
 api.add_resource(ChatMessageListApi, '/apps/<uuid:app_id>/chat-messages', endpoint='console_chat_messages')
 api.add_resource(MessageFeedbackApi, '/apps/<uuid:app_id>/feedbacks')

api/controllers/console/app/model_config.py

@@ -1,16 +1,20 @@
-# -*- coding:utf-8 -*-
+import json
 
-from controllers.console import api
-from controllers.console.app import _get_app
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
-from events.app_event import app_model_config_was_updated
-from extensions.ext_database import db
 from flask import request
 from flask_login import current_user
 from flask_restful import Resource
+
+from controllers.console import api
+from controllers.console.app.wraps import get_app_model
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from core.agent.entities import AgentToolEntity
+from core.tools.tool_manager import ToolManager
+from core.tools.utils.configuration import ToolParameterConfigurationManager
+from events.app_event import app_model_config_was_updated
+from extensions.ext_database import db
 from libs.login import login_required
-from models.model import AppModelConfig
+from models.model import AppMode, AppModelConfig
 from services.app_model_config_service import AppModelConfigService
 
 
@@ -19,33 +23,113 @@ class ModelConfigResource(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def post(self, app_id):
+    @get_app_model(mode=[AppMode.AGENT_CHAT, AppMode.CHAT, AppMode.COMPLETION])
+    def post(self, app_model):
         """Modify app model config"""
-        app_id = str(app_id)
-
-        app = _get_app(app_id)
-
         # validate config
         model_configuration = AppModelConfigService.validate_configuration(
             tenant_id=current_user.current_tenant_id,
-            account=current_user,
             config=request.json,
-            app_mode=app.mode
+            app_mode=AppMode.value_of(app_model.mode)
         )
 
         new_app_model_config = AppModelConfig(
-            app_id=app.id,
+            app_id=app_model.id,
         )
         new_app_model_config = new_app_model_config.from_model_config_dict(model_configuration)
 
+        if app_model.mode == AppMode.AGENT_CHAT.value or app_model.is_agent:
+            # get original app model config
+            original_app_model_config: AppModelConfig = db.session.query(AppModelConfig).filter(
+                AppModelConfig.id == app_model.app_model_config_id
+            ).first()
+            agent_mode = original_app_model_config.agent_mode_dict
+            # decrypt agent tool parameters if it's secret-input
+            parameter_map = {}
+            masked_parameter_map = {}
+            tool_map = {}
+            for tool in agent_mode.get('tools') or []:
+                if not isinstance(tool, dict) or len(tool.keys()) <= 3:
+                    continue
+
+                agent_tool_entity = AgentToolEntity(**tool)
+                # get tool
+                try:
+                    tool_runtime = ToolManager.get_agent_tool_runtime(
+                        tenant_id=current_user.current_tenant_id,
+                        agent_tool=agent_tool_entity,
+                    )
+                    manager = ToolParameterConfigurationManager(
+                        tenant_id=current_user.current_tenant_id,
+                        tool_runtime=tool_runtime,
+                        provider_name=agent_tool_entity.provider_id,
+                        provider_type=agent_tool_entity.provider_type,
+                    )
+                except Exception as e:
+                    continue
+
+                # get decrypted parameters
+                if agent_tool_entity.tool_parameters:
+                    parameters = manager.decrypt_tool_parameters(agent_tool_entity.tool_parameters or {})
+                    masked_parameter = manager.mask_tool_parameters(parameters or {})
+                else:
+                    parameters = {}
+                    masked_parameter = {}
+
+                key = f'{agent_tool_entity.provider_id}.{agent_tool_entity.provider_type}.{agent_tool_entity.tool_name}'
+                masked_parameter_map[key] = masked_parameter
+                parameter_map[key] = parameters
+                tool_map[key] = tool_runtime
+
+            # encrypt agent tool parameters if it's secret-input
+            agent_mode = new_app_model_config.agent_mode_dict
+            for tool in agent_mode.get('tools') or []:
+                agent_tool_entity = AgentToolEntity(**tool)
+
+                # get tool
+                key = f'{agent_tool_entity.provider_id}.{agent_tool_entity.provider_type}.{agent_tool_entity.tool_name}'
+                if key in tool_map:
+                    tool_runtime = tool_map[key]
+                else:
+                    try:
+                        tool_runtime = ToolManager.get_agent_tool_runtime(
+                            tenant_id=current_user.current_tenant_id,
+                            agent_tool=agent_tool_entity,
+                        )
+                    except Exception as e:
+                        continue
+
+                manager = ToolParameterConfigurationManager(
+                    tenant_id=current_user.current_tenant_id,
+                    tool_runtime=tool_runtime,
+                    provider_name=agent_tool_entity.provider_id,
+                    provider_type=agent_tool_entity.provider_type,
+                )
+                manager.delete_tool_parameters_cache()
+
+                # override parameters if it equals to masked parameters
+                if agent_tool_entity.tool_parameters:
+                    if key not in masked_parameter_map:
+                        continue
+
+                    if agent_tool_entity.tool_parameters == masked_parameter_map[key]:
+                        agent_tool_entity.tool_parameters = parameter_map[key]
+
+                # encrypt parameters
+                if agent_tool_entity.tool_parameters:
+                    tool['tool_parameters'] = manager.encrypt_tool_parameters(agent_tool_entity.tool_parameters or {})
+
+            # update app model config
+            new_app_model_config.agent_mode = json.dumps(agent_mode)
+
         db.session.add(new_app_model_config)
         db.session.flush()
 
-        app.app_model_config_id = new_app_model_config.id
+        app_model.app_model_config_id = new_app_model_config.id
         db.session.commit()
 
         app_model_config_was_updated.send(
-            app,
+            app_model,
             app_model_config=new_app_model_config
         )
 

api/controllers/console/app/site.py

@@ -1,16 +1,16 @@
-# -*- coding:utf-8 -*-
+from flask_login import current_user
+from flask_restful import Resource, marshal_with, reqparse
+from werkzeug.exceptions import Forbidden, NotFound
+
+from constants.languages import supported_language
 from controllers.console import api
-from controllers.console.app import _get_app
+from controllers.console.app.wraps import get_app_model
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
 from fields.app_fields import app_site_fields
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
-from libs.helper import supported_language
 from libs.login import login_required
 from models.model import Site
-from werkzeug.exceptions import Forbidden, NotFound
 
 
 def parse_app_site_args():
@@ -34,15 +34,13 @@ class AppSite(Resource):
     @setup_required
     @login_required
     @account_initialization_required
+    @get_app_model
     @marshal_with(app_site_fields)
-    def post(self, app_id):
+    def post(self, app_model):
         args = parse_app_site_args()
 
-        app_id = str(app_id)
-        app_model = _get_app(app_id)
-
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         site = db.session.query(Site). \
@@ -82,13 +80,11 @@ class AppSiteAccessTokenReset(Resource):
     @setup_required
     @login_required
     @account_initialization_required
+    @get_app_model
     @marshal_with(app_site_fields)
-    def post(self, app_id):
-        app_id = str(app_id)
-        app_model = _get_app(app_id)
-
+    def post(self, app_model):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         site = db.session.query(Site).filter(Site.app_id == app_model.id).first()

api/controllers/console/app/statistic.py

@@ -1,18 +1,19 @@
-# -*- coding:utf-8 -*-
 from datetime import datetime
 from decimal import Decimal
 
 import pytz
-from controllers.console import api
-from controllers.console.app import _get_app
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
-from extensions.ext_database import db
 from flask import jsonify
 from flask_login import current_user
 from flask_restful import Resource, reqparse
+
+from controllers.console import api
+from controllers.console.app.wraps import get_app_model
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from extensions.ext_database import db
 from libs.helper import datetime_string
 from libs.login import login_required
+from models.model import AppMode
 
 
 class DailyConversationStatistic(Resource):
@@ -20,10 +21,9 @@ class DailyConversationStatistic(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def get(self, app_id):
+    @get_app_model
+    def get(self, app_model):
         account = current_user
-        app_id = str(app_id)
-        app_model = _get_app(app_id)
 
         parser = reqparse.RequestParser()
         parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
@@ -81,10 +81,9 @@ class DailyTerminalsStatistic(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def get(self, app_id):
+    @get_app_model
+    def get(self, app_model):
         account = current_user
-        app_id = str(app_id)
-        app_model = _get_app(app_id)
 
         parser = reqparse.RequestParser()
         parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
@@ -141,10 +140,9 @@ class DailyTokenCostStatistic(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def get(self, app_id):
+    @get_app_model
+    def get(self, app_model):
         account = current_user
-        app_id = str(app_id)
-        app_model = _get_app(app_id)
 
         parser = reqparse.RequestParser()
         parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
@@ -205,10 +203,9 @@ class AverageSessionInteractionStatistic(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def get(self, app_id):
+    @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
+    def get(self, app_model):
         account = current_user
-        app_id = str(app_id)
-        app_model = _get_app(app_id, 'chat')
 
         parser = reqparse.RequestParser()
         parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
@@ -271,10 +268,9 @@ class UserSatisfactionRateStatistic(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def get(self, app_id):
+    @get_app_model
+    def get(self, app_model):
         account = current_user
-        app_id = str(app_id)
-        app_model = _get_app(app_id)
 
         parser = reqparse.RequestParser()
         parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
@@ -334,10 +330,9 @@ class AverageResponseTimeStatistic(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def get(self, app_id):
+    @get_app_model(mode=AppMode.COMPLETION)
+    def get(self, app_model):
         account = current_user
-        app_id = str(app_id)
-        app_model = _get_app(app_id, 'completion')
 
         parser = reqparse.RequestParser()
         parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
@@ -396,10 +391,9 @@ class TokensPerSecondStatistic(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def get(self, app_id):
+    @get_app_model
+    def get(self, app_model):
         account = current_user
-        app_id = str(app_id)
-        app_model = _get_app(app_id)
 
         parser = reqparse.RequestParser()
         parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')

api/controllers/console/app/workflow.py

@@ -0,0 +1,324 @@
+import json
+import logging
+
+from flask import abort, request
+from flask_restful import Resource, marshal_with, reqparse
+from werkzeug.exceptions import InternalServerError, NotFound
+
+import services
+from controllers.console import api
+from controllers.console.app.error import ConversationCompletedError, DraftWorkflowNotExist
+from controllers.console.app.wraps import get_app_model
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from core.app.apps.base_app_queue_manager import AppQueueManager
+from core.app.entities.app_invoke_entities import InvokeFrom
+from fields.workflow_fields import workflow_fields
+from fields.workflow_run_fields import workflow_run_node_execution_fields
+from libs import helper
+from libs.helper import TimestampField, uuid_value
+from libs.login import current_user, login_required
+from models.model import App, AppMode
+from services.app_generate_service import AppGenerateService
+from services.workflow_service import WorkflowService
+
+logger = logging.getLogger(__name__)
+
+
+class DraftWorkflowApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    @marshal_with(workflow_fields)
+    def get(self, app_model: App):
+        """
+        Get draft workflow
+        """
+        # fetch draft workflow by app_model
+        workflow_service = WorkflowService()
+        workflow = workflow_service.get_draft_workflow(app_model=app_model)
+
+        if not workflow:
+            raise DraftWorkflowNotExist()
+
+        # return workflow, if not found, return None (initiate graph by frontend)
+        return workflow
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    def post(self, app_model: App):
+        """
+        Sync draft workflow
+        """
+        content_type = request.headers.get('Content-Type')
+
+        if 'application/json' in content_type:
+            parser = reqparse.RequestParser()
+            parser.add_argument('graph', type=dict, required=True, nullable=False, location='json')
+            parser.add_argument('features', type=dict, required=True, nullable=False, location='json')
+            args = parser.parse_args()
+        elif 'text/plain' in content_type:
+            try:
+                data = json.loads(request.data.decode('utf-8'))
+                if 'graph' not in data or 'features' not in data:
+                    raise ValueError('graph or features not found in data')
+
+                if not isinstance(data.get('graph'), dict) or not isinstance(data.get('features'), dict):
+                    raise ValueError('graph or features is not a dict')
+
+                args = {
+                    'graph': data.get('graph'),
+                    'features': data.get('features')
+                }
+            except json.JSONDecodeError:
+                return {'message': 'Invalid JSON data'}, 400
+        else:
+            abort(415)
+
+        workflow_service = WorkflowService()
+        workflow = workflow_service.sync_draft_workflow(
+            app_model=app_model,
+            graph=args.get('graph'),
+            features=args.get('features'),
+            account=current_user
+        )
+
+        return {
+            "result": "success",
+            "updated_at": TimestampField().format(workflow.updated_at or workflow.created_at)
+        }
+
+
+class AdvancedChatDraftWorkflowRunApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT])
+    def post(self, app_model: App):
+        """
+        Run draft workflow
+        """
+        parser = reqparse.RequestParser()
+        parser.add_argument('inputs', type=dict, location='json')
+        parser.add_argument('query', type=str, required=True, location='json', default='')
+        parser.add_argument('files', type=list, location='json')
+        parser.add_argument('conversation_id', type=uuid_value, location='json')
+        args = parser.parse_args()
+
+        try:
+            response = AppGenerateService.generate(
+                app_model=app_model,
+                user=current_user,
+                args=args,
+                invoke_from=InvokeFrom.DEBUGGER,
+                streaming=True
+            )
+
+            return helper.compact_generate_response(response)
+        except services.errors.conversation.ConversationNotExistsError:
+            raise NotFound("Conversation Not Exists.")
+        except services.errors.conversation.ConversationCompletedError:
+            raise ConversationCompletedError()
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logging.exception("internal server error.")
+            raise InternalServerError()
+
+
+class DraftWorkflowRunApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.WORKFLOW])
+    def post(self, app_model: App):
+        """
+        Run draft workflow
+        """
+        parser = reqparse.RequestParser()
+        parser.add_argument('inputs', type=dict, required=True, nullable=False, location='json')
+        parser.add_argument('files', type=list, required=False, location='json')
+        args = parser.parse_args()
+
+        try:
+            response = AppGenerateService.generate(
+                app_model=app_model,
+                user=current_user,
+                args=args,
+                invoke_from=InvokeFrom.DEBUGGER,
+                streaming=True
+            )
+
+            return helper.compact_generate_response(response)
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logging.exception("internal server error.")
+            raise InternalServerError()
+
+
+class WorkflowTaskStopApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    def post(self, app_model: App, task_id: str):
+        """
+        Stop workflow task
+        """
+        AppQueueManager.set_stop_flag(task_id, InvokeFrom.DEBUGGER, current_user.id)
+
+        return {
+            "result": "success"
+        }
+
+
+class DraftWorkflowNodeRunApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    @marshal_with(workflow_run_node_execution_fields)
+    def post(self, app_model: App, node_id: str):
+        """
+        Run draft workflow node
+        """
+        parser = reqparse.RequestParser()
+        parser.add_argument('inputs', type=dict, required=True, nullable=False, location='json')
+        args = parser.parse_args()
+
+        workflow_service = WorkflowService()
+        workflow_node_execution = workflow_service.run_draft_workflow_node(
+            app_model=app_model,
+            node_id=node_id,
+            user_inputs=args.get('inputs'),
+            account=current_user
+        )
+
+        return workflow_node_execution
+
+
+class PublishedWorkflowApi(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    @marshal_with(workflow_fields)
+    def get(self, app_model: App):
+        """
+        Get published workflow
+        """
+        # fetch published workflow by app_model
+        workflow_service = WorkflowService()
+        workflow = workflow_service.get_published_workflow(app_model=app_model)
+
+        # return workflow, if not found, return None
+        return workflow
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    def post(self, app_model: App):
+        """
+        Publish workflow
+        """
+        workflow_service = WorkflowService()
+        workflow = workflow_service.publish_workflow(app_model=app_model, account=current_user)
+
+        return {
+            "result": "success",
+            "created_at": TimestampField().format(workflow.created_at)
+        }
+
+
+class DefaultBlockConfigsApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    def get(self, app_model: App):
+        """
+        Get default block config
+        """
+        # Get default block configs
+        workflow_service = WorkflowService()
+        return workflow_service.get_default_block_configs()
+
+
+class DefaultBlockConfigApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    def get(self, app_model: App, block_type: str):
+        """
+        Get default block config
+        """
+        parser = reqparse.RequestParser()
+        parser.add_argument('q', type=str, location='args')
+        args = parser.parse_args()
+
+        filters = None
+        if args.get('q'):
+            try:
+                filters = json.loads(args.get('q'))
+            except json.JSONDecodeError:
+                raise ValueError('Invalid filters')
+
+        # Get default block configs
+        workflow_service = WorkflowService()
+        return workflow_service.get_default_block_config(
+            node_type=block_type,
+            filters=filters
+        )
+
+
+class ConvertToWorkflowApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.CHAT, AppMode.COMPLETION])
+    def post(self, app_model: App):
+        """
+        Convert basic mode of chatbot app to workflow mode
+        Convert expert mode of chatbot app to workflow mode
+        Convert Completion App to Workflow App
+        """
+        if request.data:
+            parser = reqparse.RequestParser()
+            parser.add_argument('name', type=str, required=False, nullable=True, location='json')
+            parser.add_argument('icon', type=str, required=False, nullable=True, location='json')
+            parser.add_argument('icon_background', type=str, required=False, nullable=True, location='json')
+            args = parser.parse_args()
+        else:
+            args = {}
+
+        # convert to workflow mode
+        workflow_service = WorkflowService()
+        new_app_model = workflow_service.convert_to_workflow(
+            app_model=app_model,
+            account=current_user,
+            args=args
+        )
+
+        # return app id
+        return {
+            'new_app_id': new_app_model.id,
+        }
+
+
+api.add_resource(DraftWorkflowApi, '/apps/<uuid:app_id>/workflows/draft')
+api.add_resource(AdvancedChatDraftWorkflowRunApi, '/apps/<uuid:app_id>/advanced-chat/workflows/draft/run')
+api.add_resource(DraftWorkflowRunApi, '/apps/<uuid:app_id>/workflows/draft/run')
+api.add_resource(WorkflowTaskStopApi, '/apps/<uuid:app_id>/workflow-runs/tasks/<string:task_id>/stop')
+api.add_resource(DraftWorkflowNodeRunApi, '/apps/<uuid:app_id>/workflows/draft/nodes/<string:node_id>/run')
+api.add_resource(PublishedWorkflowApi, '/apps/<uuid:app_id>/workflows/publish')
+api.add_resource(DefaultBlockConfigsApi, '/apps/<uuid:app_id>/workflows/default-workflow-block-configs')
+api.add_resource(DefaultBlockConfigApi, '/apps/<uuid:app_id>/workflows/default-workflow-block-configs'
+                                        '/<string:block_type>')
+api.add_resource(ConvertToWorkflowApi, '/apps/<uuid:app_id>/convert-to-workflow')

api/controllers/console/app/workflow_app_log.py

@@ -0,0 +1,41 @@
+from flask_restful import Resource, marshal_with, reqparse
+from flask_restful.inputs import int_range
+
+from controllers.console import api
+from controllers.console.app.wraps import get_app_model
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from fields.workflow_app_log_fields import workflow_app_log_pagination_fields
+from libs.login import login_required
+from models.model import App, AppMode
+from services.workflow_app_service import WorkflowAppService
+
+
+class WorkflowAppLogApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.WORKFLOW])
+    @marshal_with(workflow_app_log_pagination_fields)
+    def get(self, app_model: App):
+        """
+        Get workflow app logs
+        """
+        parser = reqparse.RequestParser()
+        parser.add_argument('keyword', type=str, location='args')
+        parser.add_argument('status', type=str, choices=['succeeded', 'failed', 'stopped'], location='args')
+        parser.add_argument('page', type=int_range(1, 99999), default=1, location='args')
+        parser.add_argument('limit', type=int_range(1, 100), default=20, location='args')
+        args = parser.parse_args()
+
+        # get paginate workflow app logs
+        workflow_app_service = WorkflowAppService()
+        workflow_app_log_pagination = workflow_app_service.get_paginate_workflow_app_logs(
+            app_model=app_model,
+            args=args
+        )
+
+        return workflow_app_log_pagination
+
+
+api.add_resource(WorkflowAppLogApi, '/apps/<uuid:app_id>/workflow-app-logs')

api/controllers/console/app/workflow_run.py

@@ -0,0 +1,109 @@
+from flask_restful import Resource, marshal_with, reqparse
+from flask_restful.inputs import int_range
+
+from controllers.console import api
+from controllers.console.app.wraps import get_app_model
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from fields.workflow_run_fields import (
+    advanced_chat_workflow_run_pagination_fields,
+    workflow_run_detail_fields,
+    workflow_run_node_execution_list_fields,
+    workflow_run_pagination_fields,
+)
+from libs.helper import uuid_value
+from libs.login import login_required
+from models.model import App, AppMode
+from services.workflow_run_service import WorkflowRunService
+
+
+class AdvancedChatAppWorkflowRunListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT])
+    @marshal_with(advanced_chat_workflow_run_pagination_fields)
+    def get(self, app_model: App):
+        """
+        Get advanced chat app workflow run list
+        """
+        parser = reqparse.RequestParser()
+        parser.add_argument('last_id', type=uuid_value, location='args')
+        parser.add_argument('limit', type=int_range(1, 100), required=False, default=20, location='args')
+        args = parser.parse_args()
+
+        workflow_run_service = WorkflowRunService()
+        result = workflow_run_service.get_paginate_advanced_chat_workflow_runs(
+            app_model=app_model,
+            args=args
+        )
+
+        return result
+
+
+class WorkflowRunListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    @marshal_with(workflow_run_pagination_fields)
+    def get(self, app_model: App):
+        """
+        Get workflow run list
+        """
+        parser = reqparse.RequestParser()
+        parser.add_argument('last_id', type=uuid_value, location='args')
+        parser.add_argument('limit', type=int_range(1, 100), required=False, default=20, location='args')
+        args = parser.parse_args()
+
+        workflow_run_service = WorkflowRunService()
+        result = workflow_run_service.get_paginate_workflow_runs(
+            app_model=app_model,
+            args=args
+        )
+
+        return result
+
+
+class WorkflowRunDetailApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    @marshal_with(workflow_run_detail_fields)
+    def get(self, app_model: App, run_id):
+        """
+        Get workflow run detail
+        """
+        run_id = str(run_id)
+
+        workflow_run_service = WorkflowRunService()
+        workflow_run = workflow_run_service.get_workflow_run(app_model=app_model, run_id=run_id)
+
+        return workflow_run
+
+
+class WorkflowRunNodeExecutionListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    @marshal_with(workflow_run_node_execution_list_fields)
+    def get(self, app_model: App, run_id):
+        """
+        Get workflow run node execution list
+        """
+        run_id = str(run_id)
+
+        workflow_run_service = WorkflowRunService()
+        node_executions = workflow_run_service.get_workflow_run_node_executions(app_model=app_model, run_id=run_id)
+
+        return {
+            'data': node_executions
+        }
+
+
+api.add_resource(AdvancedChatAppWorkflowRunListApi, '/apps/<uuid:app_id>/advanced-chat/workflow-runs')
+api.add_resource(WorkflowRunListApi, '/apps/<uuid:app_id>/workflow-runs')
+api.add_resource(WorkflowRunDetailApi, '/apps/<uuid:app_id>/workflow-runs/<uuid:run_id>')
+api.add_resource(WorkflowRunNodeExecutionListApi, '/apps/<uuid:app_id>/workflow-runs/<uuid:run_id>/node-executions')

api/controllers/console/app/workflow_statistic.py

@@ -0,0 +1,278 @@
+from datetime import datetime
+from decimal import Decimal
+
+import pytz
+from flask import jsonify
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+
+from controllers.console import api
+from controllers.console.app.wraps import get_app_model
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from extensions.ext_database import db
+from libs.helper import datetime_string
+from libs.login import login_required
+from models.model import AppMode
+from models.workflow import WorkflowRunTriggeredFrom
+
+
+class WorkflowDailyRunsStatistic(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model
+    def get(self, app_model):
+        account = current_user
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        parser.add_argument('end', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        args = parser.parse_args()
+
+        sql_query = '''
+        SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, count(id) AS runs
+            FROM workflow_runs 
+            WHERE app_id = :app_id 
+                AND triggered_from = :triggered_from
+        '''
+        arg_dict = {'tz': account.timezone, 'app_id': app_model.id, 'triggered_from': WorkflowRunTriggeredFrom.APP_RUN.value}
+
+        timezone = pytz.timezone(account.timezone)
+        utc_timezone = pytz.utc
+
+        if args['start']:
+            start_datetime = datetime.strptime(args['start'], '%Y-%m-%d %H:%M')
+            start_datetime = start_datetime.replace(second=0)
+
+            start_datetime_timezone = timezone.localize(start_datetime)
+            start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at >= :start'
+            arg_dict['start'] = start_datetime_utc
+
+        if args['end']:
+            end_datetime = datetime.strptime(args['end'], '%Y-%m-%d %H:%M')
+            end_datetime = end_datetime.replace(second=0)
+
+            end_datetime_timezone = timezone.localize(end_datetime)
+            end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at < :end'
+            arg_dict['end'] = end_datetime_utc
+
+        sql_query += ' GROUP BY date order by date'
+
+        response_data = []
+
+        with db.engine.begin() as conn:
+            rs = conn.execute(db.text(sql_query), arg_dict)
+            for i in rs:
+                response_data.append({
+                    'date': str(i.date),
+                    'runs': i.runs
+                })
+
+        return jsonify({
+            'data': response_data
+        })
+
+class WorkflowDailyTerminalsStatistic(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model
+    def get(self, app_model):
+        account = current_user
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        parser.add_argument('end', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        args = parser.parse_args()
+
+        sql_query = '''
+                SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, count(distinct workflow_runs.created_by) AS terminal_count
+                    FROM workflow_runs 
+                    WHERE app_id = :app_id 
+                        AND triggered_from = :triggered_from
+                '''
+        arg_dict = {'tz': account.timezone, 'app_id': app_model.id, 'triggered_from': WorkflowRunTriggeredFrom.APP_RUN.value}
+
+        timezone = pytz.timezone(account.timezone)
+        utc_timezone = pytz.utc
+
+        if args['start']:
+            start_datetime = datetime.strptime(args['start'], '%Y-%m-%d %H:%M')
+            start_datetime = start_datetime.replace(second=0)
+
+            start_datetime_timezone = timezone.localize(start_datetime)
+            start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at >= :start'
+            arg_dict['start'] = start_datetime_utc
+
+        if args['end']:
+            end_datetime = datetime.strptime(args['end'], '%Y-%m-%d %H:%M')
+            end_datetime = end_datetime.replace(second=0)
+
+            end_datetime_timezone = timezone.localize(end_datetime)
+            end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at < :end'
+            arg_dict['end'] = end_datetime_utc
+
+        sql_query += ' GROUP BY date order by date'
+
+        response_data = []
+
+        with db.engine.begin() as conn:
+            rs = conn.execute(db.text(sql_query), arg_dict)            
+            for i in rs:
+                response_data.append({
+                    'date': str(i.date),
+                    'terminal_count': i.terminal_count
+                })
+
+        return jsonify({
+            'data': response_data
+        })
+
+class WorkflowDailyTokenCostStatistic(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model
+    def get(self, app_model):
+        account = current_user
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        parser.add_argument('end', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        args = parser.parse_args()
+
+        sql_query = '''
+                SELECT 
+                    date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, 
+                    SUM(workflow_runs.total_tokens) as token_count
+                FROM workflow_runs 
+                WHERE app_id = :app_id 
+                    AND triggered_from = :triggered_from
+                '''
+        arg_dict = {'tz': account.timezone, 'app_id': app_model.id, 'triggered_from': WorkflowRunTriggeredFrom.APP_RUN.value}
+
+        timezone = pytz.timezone(account.timezone)
+        utc_timezone = pytz.utc
+
+        if args['start']:
+            start_datetime = datetime.strptime(args['start'], '%Y-%m-%d %H:%M')
+            start_datetime = start_datetime.replace(second=0)
+
+            start_datetime_timezone = timezone.localize(start_datetime)
+            start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at >= :start'
+            arg_dict['start'] = start_datetime_utc
+
+        if args['end']:
+            end_datetime = datetime.strptime(args['end'], '%Y-%m-%d %H:%M')
+            end_datetime = end_datetime.replace(second=0)
+
+            end_datetime_timezone = timezone.localize(end_datetime)
+            end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at < :end'
+            arg_dict['end'] = end_datetime_utc
+
+        sql_query += ' GROUP BY date order by date'
+
+        response_data = []
+
+        with db.engine.begin() as conn:
+            rs = conn.execute(db.text(sql_query), arg_dict)
+            for i in rs:
+                response_data.append({
+                    'date': str(i.date),
+                    'token_count': i.token_count,
+                })
+
+        return jsonify({
+            'data': response_data
+        })
+
+class WorkflowAverageAppInteractionStatistic(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.WORKFLOW])
+    def get(self, app_model):
+        account = current_user
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        parser.add_argument('end', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        args = parser.parse_args()
+
+        sql_query = """
+            SELECT 
+                AVG(sub.interactions) as interactions,
+                sub.date
+            FROM
+                (SELECT 
+                    date(DATE_TRUNC('day', c.created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, 
+                    c.created_by,
+                    COUNT(c.id) AS interactions
+                FROM workflow_runs c
+                WHERE c.app_id = :app_id
+                    AND c.triggered_from = :triggered_from
+                    {{start}}
+                    {{end}}
+                GROUP BY date, c.created_by) sub
+            GROUP BY sub.created_by, sub.date
+            """
+        arg_dict = {'tz': account.timezone, 'app_id': app_model.id, 'triggered_from': WorkflowRunTriggeredFrom.APP_RUN.value}
+
+        timezone = pytz.timezone(account.timezone)
+        utc_timezone = pytz.utc
+
+        if args['start']:
+            start_datetime = datetime.strptime(args['start'], '%Y-%m-%d %H:%M')
+            start_datetime = start_datetime.replace(second=0)
+
+            start_datetime_timezone = timezone.localize(start_datetime)
+            start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query = sql_query.replace('{{start}}', ' AND c.created_at >= :start')
+            arg_dict['start'] = start_datetime_utc
+        else:
+            sql_query = sql_query.replace('{{start}}', '')
+
+        if args['end']:
+            end_datetime = datetime.strptime(args['end'], '%Y-%m-%d %H:%M')
+            end_datetime = end_datetime.replace(second=0)
+
+            end_datetime_timezone = timezone.localize(end_datetime)
+            end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query = sql_query.replace('{{end}}', ' and c.created_at < :end')
+            arg_dict['end'] = end_datetime_utc
+        else:
+            sql_query = sql_query.replace('{{end}}', '')
+
+        response_data = []
+        
+        with db.engine.begin() as conn:
+            rs = conn.execute(db.text(sql_query), arg_dict)
+            for i in rs:
+                response_data.append({
+                    'date': str(i.date),
+                    'interactions': float(i.interactions.quantize(Decimal('0.01')))
+                })
+
+        return jsonify({
+            'data': response_data
+        })
+
+api.add_resource(WorkflowDailyRunsStatistic, '/apps/<uuid:app_id>/workflow/statistics/daily-conversations')
+api.add_resource(WorkflowDailyTerminalsStatistic, '/apps/<uuid:app_id>/workflow/statistics/daily-terminals')
+api.add_resource(WorkflowDailyTokenCostStatistic, '/apps/<uuid:app_id>/workflow/statistics/token-costs')
+api.add_resource(WorkflowAverageAppInteractionStatistic, '/apps/<uuid:app_id>/workflow/statistics/average-app-interactions')

api/controllers/console/app/wraps.py

@@ -0,0 +1,55 @@
+from collections.abc import Callable
+from functools import wraps
+from typing import Optional, Union
+
+from controllers.console.app.error import AppNotFoundError
+from extensions.ext_database import db
+from libs.login import current_user
+from models.model import App, AppMode
+
+
+def get_app_model(view: Optional[Callable] = None, *,
+                  mode: Union[AppMode, list[AppMode]] = None):
+    def decorator(view_func):
+        @wraps(view_func)
+        def decorated_view(*args, **kwargs):
+            if not kwargs.get('app_id'):
+                raise ValueError('missing app_id in path parameters')
+
+            app_id = kwargs.get('app_id')
+            app_id = str(app_id)
+
+            del kwargs['app_id']
+
+            app_model = db.session.query(App).filter(
+                App.id == app_id,
+                App.tenant_id == current_user.current_tenant_id,
+                App.status == 'normal'
+            ).first()
+
+            if not app_model:
+                raise AppNotFoundError()
+
+            app_mode = AppMode.value_of(app_model.mode)
+            if app_mode == AppMode.CHANNEL:
+                raise AppNotFoundError()
+
+            if mode is not None:
+                if isinstance(mode, list):
+                    modes = mode
+                else:
+                    modes = [mode]
+
+                if app_mode not in modes:
+                    mode_values = {m.value for m in modes}
+                    raise AppNotFoundError(f"App mode is not in the supported list: {mode_values}")
+
+            kwargs['app_model'] = app_model
+
+            return view_func(*args, **kwargs)
+        return decorated_view
+
+    if view is None:
+        return decorator
+    else:
+        return decorator(view)

api/controllers/console/auth/activate.py

@@ -2,13 +2,15 @@ import base64
 import secrets
 from datetime import datetime
 
+from flask_restful import Resource, reqparse
+
+from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.error import AlreadyActivateError
 from extensions.ext_database import db
-from flask_restful import Resource, reqparse
-from libs.helper import email, str_len, supported_language, timezone
+from libs.helper import email, str_len, timezone
 from libs.password import hash_password, valid_password
-from models.account import AccountStatus, Tenant
+from models.account import AccountStatus
 from services.account_service import RegisterService
 
 

api/controllers/console/auth/data_source_oauth.py

@@ -1,13 +1,14 @@
 import logging
 
 import requests
-from controllers.console import api
 from flask import current_app, redirect, request
 from flask_login import current_user
 from flask_restful import Resource
+from werkzeug.exceptions import Forbidden
+
+from controllers.console import api
 from libs.login import login_required
 from libs.oauth_data_source import NotionOAuth
-from werkzeug.exceptions import Forbidden
 
 from ..setup import setup_required
 from ..wraps import account_initialization_required
@@ -30,7 +31,7 @@ def get_oauth_providers():
 class OAuthDataSource(Resource):
     def get(self, provider: str):
         # The role of the current user in the table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
         OAUTH_DATASOURCE_PROVIDERS = get_oauth_providers()
         with current_app.app_context():

api/controllers/console/auth/login.py

@@ -1,11 +1,10 @@
-# -*- coding:utf-8 -*-
-import flask
 import flask_login
+from flask import current_app, request
+from flask_restful import Resource, reqparse
+
 import services
 from controllers.console import api
 from controllers.console.setup import setup_required
-from flask import current_app, request
-from flask_restful import Resource, reqparse
 from libs.helper import email
 from libs.password import valid_password
 from services.account_service import AccountService, TenantService
@@ -30,10 +29,7 @@ class LoginApi(Resource):
         except services.errors.account.AccountLoginError:
             return {'code': 'unauthorized', 'message': 'Invalid email or password'}, 401
 
-        try:
-            TenantService.switch_tenant(account)
-        except Exception:
-            pass
+        TenantService.create_owner_tenant_if_not_exist(account)
 
         AccountService.update_last_login(account, request)
 
@@ -47,7 +43,6 @@ class LogoutApi(Resource):
 
     @setup_required
     def get(self):
-        flask.session.pop('workspace_id', None)
         flask_login.logout_user()
         return {'result': 'success'}
 

api/controllers/console/auth/oauth.py

@@ -3,12 +3,14 @@ from datetime import datetime
 from typing import Optional
 
 import requests
-from extensions.ext_database import db
 from flask import current_app, redirect, request
 from flask_restful import Resource
+
+from constants.languages import languages
+from extensions.ext_database import db
 from libs.oauth import GitHubOAuth, GoogleOAuth, OAuthUserInfo
 from models.account import Account, AccountStatus
-from services.account_service import AccountService, RegisterService
+from services.account_service import AccountService, RegisterService, TenantService
 
 from .. import api
 
@@ -74,6 +76,8 @@ class OAuthCallback(Resource):
             account.initialized_at = datetime.utcnow()
             db.session.commit()
 
+        TenantService.create_owner_tenant_if_not_exist(account)
+
         AccountService.update_last_login(account, request)
 
         token = AccountService.get_account_jwt_token(account)
@@ -106,11 +110,11 @@ def _generate_account(provider: str, user_info: OAuthUserInfo):
         )
 
         # Set interface language
-        preferred_lang = request.accept_languages.best_match(['zh', 'en'])
-        if preferred_lang == 'zh':
-            interface_language = 'zh-Hans'
+        preferred_lang = request.accept_languages.best_match(languages)
+        if preferred_lang and preferred_lang in languages:
+            interface_language = preferred_lang
         else:
-            interface_language = 'en-US'
+            interface_language = languages[0]
         account.interface_language = interface_language
         db.session.commit()
 

api/controllers/console/billing/billing.py

@@ -1,8 +1,9 @@
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, only_edition_cloud
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 from services.billing_service import BillingService
 
@@ -20,7 +21,7 @@ class Subscription(Resource):
         parser.add_argument('interval', type=str, required=True, location='args', choices=['month', 'year'])
         args = parser.parse_args()
 
-        BillingService.is_tenant_owner(current_user)
+        BillingService.is_tenant_owner_or_admin(current_user)
 
         return BillingService.get_subscription(args['plan'],
                                                args['interval'],
@@ -35,8 +36,8 @@ class Invoices(Resource):
     @account_initialization_required
     @only_edition_cloud
     def get(self):
-        BillingService.is_tenant_owner(current_user)
-        return BillingService.get_invoices(current_user.email)
+        BillingService.is_tenant_owner_or_admin(current_user)
+        return BillingService.get_invoices(current_user.email, current_user.current_tenant_id)
 
 
 api.add_resource(Subscription, '/billing/subscription')

api/controllers/console/datasets/data_source.py

@@ -1,22 +1,24 @@
 import datetime
 import json
 
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, marshal_with, reqparse
+from werkzeug.exceptions import NotFound
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
-from core.data_loader.loader.notion import NotionLoader
 from core.indexing_runner import IndexingRunner
+from core.rag.extractor.entity.extract_setting import ExtractSetting
+from core.rag.extractor.notion_extractor import NotionExtractor
 from extensions.ext_database import db
 from fields.data_source_fields import integrate_list_fields, integrate_notion_info_list_fields
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
 from libs.login import login_required
 from models.dataset import Document
 from models.source import DataSourceBinding
 from services.dataset_service import DatasetService, DocumentService
 from tasks.document_indexing_sync_task import document_indexing_sync_task
-from werkzeug.exceptions import NotFound
 
 
 class DataSourceApi(Resource):
@@ -172,14 +174,15 @@ class DataSourceNotionApi(Resource):
         if not data_source_binding:
             raise NotFound('Data source binding not found.')
 
-        loader = NotionLoader(
-            notion_access_token=data_source_binding.access_token,
+        extractor = NotionExtractor(
             notion_workspace_id=workspace_id,
             notion_obj_id=page_id,
-            notion_page_type=page_type
+            notion_page_type=page_type,
+            notion_access_token=data_source_binding.access_token,
+            tenant_id=current_user.current_tenant_id
         )
 
-        text_docs = loader.load()
+        text_docs = extractor.extract()
         return {
             'content': "\n".join([doc.page_content for doc in text_docs])
         }, 200
@@ -191,11 +194,31 @@ class DataSourceNotionApi(Resource):
         parser = reqparse.RequestParser()
         parser.add_argument('notion_info_list', type=list, required=True, nullable=True, location='json')
         parser.add_argument('process_rule', type=dict, required=True, nullable=True, location='json')
+        parser.add_argument('doc_form', type=str, default='text_model', required=False, nullable=False, location='json')
+        parser.add_argument('doc_language', type=str, default='English', required=False, nullable=False, location='json')
         args = parser.parse_args()
         # validate args
         DocumentService.estimate_args_validate(args)
+        notion_info_list = args['notion_info_list']
+        extract_settings = []
+        for notion_info in notion_info_list:
+            workspace_id = notion_info['workspace_id']
+            for page in notion_info['pages']:
+                extract_setting = ExtractSetting(
+                    datasource_type="notion_import",
+                    notion_info={
+                        "notion_workspace_id": workspace_id,
+                        "notion_obj_id": page['page_id'],
+                        "notion_page_type": page['type'],
+                        "tenant_id": current_user.current_tenant_id
+                    },
+                    document_model=args['doc_form']
+                )
+                extract_settings.append(extract_setting)
         indexing_runner = IndexingRunner()
-        response = indexing_runner.notion_indexing_estimate(current_user.current_tenant_id, args['notion_info_list'], args['process_rule'])
+        response = indexing_runner.indexing_estimate(current_user.current_tenant_id, extract_settings,
+                                                     args['process_rule'], args['doc_form'],
+                                                     args['doc_language'])
         return response, 200
 
 

api/controllers/console/datasets/datasets.py

@@ -1,5 +1,9 @@
-# -*- coding:utf-8 -*-
 import flask_restful
+from flask import current_app, request
+from flask_login import current_user
+from flask_restful import Resource, marshal, marshal_with, reqparse
+from werkzeug.exceptions import Forbidden, NotFound
+
 import services
 from controllers.console import api
 from controllers.console.apikey import api_key_fields, api_key_list
@@ -11,18 +15,15 @@ from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.indexing_runner import IndexingRunner
 from core.model_runtime.entities.model_entities import ModelType
 from core.provider_manager import ProviderManager
+from core.rag.extractor.entity.extract_setting import ExtractSetting
 from extensions.ext_database import db
 from fields.app_fields import related_app_list
 from fields.dataset_fields import dataset_detail_fields, dataset_query_detail_fields
 from fields.document_fields import document_status_fields
-from flask import current_app, request
-from flask_login import current_user
-from flask_restful import Resource, marshal, marshal_with, reqparse
 from libs.login import login_required
-from models.dataset import Document, DocumentSegment
+from models.dataset import Dataset, Document, DocumentSegment
 from models.model import ApiToken, UploadFile
 from services.dataset_service import DatasetService, DocumentService
-from werkzeug.exceptions import Forbidden, NotFound
 
 
 def _validate_name(name):
@@ -97,12 +98,13 @@ class DatasetListApi(Resource):
                             help='type is required. Name must be between 1 to 40 characters.',
                             type=_validate_name)
         parser.add_argument('indexing_technique', type=str, location='json',
-                            choices=('high_quality', 'economy'),
+                            choices=Dataset.INDEXING_TECHNIQUE_LIST,
+                            nullable=True,
                             help='Invalid indexing technique.')
         args = parser.parse_args()
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         try:
@@ -177,7 +179,8 @@ class DatasetApi(Resource):
                             location='json', store_missing=False,
                             type=_validate_description_length)
         parser.add_argument('indexing_technique', type=str, location='json',
-                            choices=('high_quality', 'economy'),
+                            choices=Dataset.INDEXING_TECHNIQUE_LIST,
+                            nullable=True,
                             help='Invalid indexing technique.')
         parser.add_argument('permission', type=str, location='json', choices=(
             'only_me', 'all_team_members'), help='Invalid permission.')
@@ -185,7 +188,7 @@ class DatasetApi(Resource):
         args = parser.parse_args()
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         dataset = DatasetService.update_dataset(
@@ -203,7 +206,7 @@ class DatasetApi(Resource):
         dataset_id_str = str(dataset_id)
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         if DatasetService.delete_dataset(dataset_id_str, current_user):
@@ -256,7 +259,9 @@ class DatasetIndexingEstimateApi(Resource):
         parser = reqparse.RequestParser()
         parser.add_argument('info_list', type=dict, required=True, nullable=True, location='json')
         parser.add_argument('process_rule', type=dict, required=True, nullable=True, location='json')
-        parser.add_argument('indexing_technique', type=str, required=True, nullable=True, location='json')
+        parser.add_argument('indexing_technique', type=str, required=True,
+                            choices=Dataset.INDEXING_TECHNIQUE_LIST,
+                            nullable=True, location='json')
         parser.add_argument('doc_form', type=str, default='text_model', required=False, nullable=False, location='json')
         parser.add_argument('dataset_id', type=str, required=False, nullable=False, location='json')
         parser.add_argument('doc_language', type=str, default='English', required=False, nullable=False,
@@ -264,6 +269,7 @@ class DatasetIndexingEstimateApi(Resource):
         args = parser.parse_args()
         # validate args
         DocumentService.estimate_args_validate(args)
+        extract_settings = []
         if args['info_list']['data_source_type'] == 'upload_file':
             file_ids = args['info_list']['file_info_list']['file_ids']
             file_details = db.session.query(UploadFile).filter(
@@ -274,37 +280,45 @@ class DatasetIndexingEstimateApi(Resource):
             if file_details is None:
                 raise NotFound("File not found.")
 
-            indexing_runner = IndexingRunner()
-
-            try:
-                response = indexing_runner.file_indexing_estimate(current_user.current_tenant_id, file_details,
-                                                                  args['process_rule'], args['doc_form'],
-                                                                  args['doc_language'], args['dataset_id'],
-                                                                  args['indexing_technique'])
-            except LLMBadRequestError:
-                raise ProviderNotInitializeError(
-                    f"No Embedding Model available. Please configure a valid provider "
-                    f"in the Settings -> Model Provider.")
-            except ProviderTokenNotInitError as ex:
-                raise ProviderNotInitializeError(ex.description)
+            if file_details:
+                for file_detail in file_details:
+                    extract_setting = ExtractSetting(
+                        datasource_type="upload_file",
+                        upload_file=file_detail,
+                        document_model=args['doc_form']
+                    )
+                    extract_settings.append(extract_setting)
         elif args['info_list']['data_source_type'] == 'notion_import':
-
-            indexing_runner = IndexingRunner()
-
-            try:
-                response = indexing_runner.notion_indexing_estimate(current_user.current_tenant_id,
-                                                                    args['info_list']['notion_info_list'],
-                                                                    args['process_rule'], args['doc_form'],
-                                                                    args['doc_language'], args['dataset_id'],
-                                                                    args['indexing_technique'])
-            except LLMBadRequestError:
-                raise ProviderNotInitializeError(
-                    f"No Embedding Model available. Please configure a valid provider "
-                    f"in the Settings -> Model Provider.")
-            except ProviderTokenNotInitError as ex:
-                raise ProviderNotInitializeError(ex.description)
+            notion_info_list = args['info_list']['notion_info_list']
+            for notion_info in notion_info_list:
+                workspace_id = notion_info['workspace_id']
+                for page in notion_info['pages']:
+                    extract_setting = ExtractSetting(
+                        datasource_type="notion_import",
+                        notion_info={
+                            "notion_workspace_id": workspace_id,
+                            "notion_obj_id": page['page_id'],
+                            "notion_page_type": page['type'],
+                            "tenant_id": current_user.current_tenant_id
+                        },
+                        document_model=args['doc_form']
+                    )
+                    extract_settings.append(extract_setting)
         else:
             raise ValueError('Data source type not support')
+        indexing_runner = IndexingRunner()
+        try:
+            response = indexing_runner.indexing_estimate(current_user.current_tenant_id, extract_settings,
+                                                         args['process_rule'], args['doc_form'],
+                                                         args['doc_language'], args['dataset_id'],
+                                                         args['indexing_technique'])
+        except LLMBadRequestError:
+            raise ProviderNotInitializeError(
+                "No Embedding Model available. Please configure a valid provider "
+                "in the Settings -> Model Provider.")
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+
         return response, 200
 
 
@@ -387,7 +401,7 @@ class DatasetApiKeyApi(Resource):
     @marshal_with(api_key_fields)
     def post(self):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         current_key_count = db.session.query(ApiToken). \
@@ -421,7 +435,7 @@ class DatasetApiDeleteApi(Resource):
         api_key_id = str(api_key_id)
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         key = db.session.query(ApiToken). \
@@ -504,4 +518,3 @@ api.add_resource(DatasetApiDeleteApi, '/datasets/api-keys/<uuid:api_key_id>')
 api.add_resource(DatasetApiBaseUrlApi, '/datasets/api-base-info')
 api.add_resource(DatasetRetrievalSettingApi, '/datasets/retrieval-setting')
 api.add_resource(DatasetRetrievalSettingMockApi, '/datasets/retrieval-setting/<string:vector_type>')
-

api/controllers/console/datasets/datasets_document.py

@@ -1,36 +1,52 @@
-# -*- coding:utf-8 -*-
 from datetime import datetime
-from typing import List
+
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, fields, marshal, marshal_with, reqparse
+from sqlalchemy import asc, desc
+from werkzeug.exceptions import Forbidden, NotFound
 
 import services
 from controllers.console import api
-from controllers.console.app.error import (ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
-from controllers.console.datasets.error import (ArchivedDocumentImmutableError, DocumentAlreadyFinishedError,
-                                                DocumentIndexingError, InvalidActionError, InvalidMetadataError)
+from controllers.console.app.error import (
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
+from controllers.console.datasets.error import (
+    ArchivedDocumentImmutableError,
+    DocumentAlreadyFinishedError,
+    DocumentIndexingError,
+    InvalidActionError,
+    InvalidMetadataError,
+)
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
-from core.errors.error import (LLMBadRequestError, ModelCurrentlyNotSupportError, ProviderTokenNotInitError,
-                               QuotaExceededError)
+from core.errors.error import (
+    LLMBadRequestError,
+    ModelCurrentlyNotSupportError,
+    ProviderTokenNotInitError,
+    QuotaExceededError,
+)
 from core.indexing_runner import IndexingRunner
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.invoke import InvokeAuthorizationError
+from core.rag.extractor.entity.extract_setting import ExtractSetting
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
-from fields.document_fields import (dataset_and_document_fields, document_fields, document_status_fields,
-                                    document_with_segments_fields)
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, fields, marshal, marshal_with, reqparse
+from fields.document_fields import (
+    dataset_and_document_fields,
+    document_fields,
+    document_status_fields,
+    document_with_segments_fields,
+)
 from libs.login import login_required
 from models.dataset import Dataset, DatasetProcessRule, Document, DocumentSegment
 from models.model import UploadFile
 from services.dataset_service import DatasetService, DocumentService
-from sqlalchemy import asc, desc
 from tasks.add_document_to_index_task import add_document_to_index_task
 from tasks.remove_document_from_index_task import remove_document_from_index_task
-from werkzeug.exceptions import Forbidden, NotFound
 
 
 class DocumentResource(Resource):
@@ -54,7 +70,7 @@ class DocumentResource(Resource):
 
         return document
 
-    def get_batch_documents(self, dataset_id: str, batch: str) -> List[Document]:
+    def get_batch_documents(self, dataset_id: str, batch: str) -> list[Document]:
         dataset = DatasetService.get_dataset(dataset_id)
         if not dataset:
             raise NotFound('Dataset not found.')
@@ -80,7 +96,7 @@ class GetProcessRuleApi(Resource):
         req_data = request.args
 
         document_id = req_data.get('document_id')
-        
+
         # get default rules
         mode = DocumentService.DEFAULT_RULES['mode']
         rules = DocumentService.DEFAULT_RULES['rules']
@@ -204,7 +220,7 @@ class DatasetDocumentListApi(Resource):
             raise NotFound('Dataset not found.')
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         try:
@@ -256,7 +272,7 @@ class DatasetInitApi(Resource):
     @cloud_edition_billing_resource_check('vector_space')
     def post(self):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -279,8 +295,8 @@ class DatasetInitApi(Resource):
                 )
             except InvokeAuthorizationError:
                 raise ProviderNotInitializeError(
-                    f"No Embedding Model available. Please configure a valid provider "
-                    f"in the Settings -> Model Provider.")
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider.")
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
 
@@ -347,16 +363,22 @@ class DocumentIndexingEstimateApi(DocumentResource):
                 if not file:
                     raise NotFound('File not found.')
 
+                extract_setting = ExtractSetting(
+                    datasource_type="upload_file",
+                    upload_file=file,
+                    document_model=document.doc_form
+                )
+
                 indexing_runner = IndexingRunner()
 
                 try:
-                    response = indexing_runner.file_indexing_estimate(current_user.current_tenant_id, [file],
-                                                                      data_process_rule_dict, None,
-                                                                      'English', dataset_id)
+                    response = indexing_runner.indexing_estimate(current_user.current_tenant_id, [extract_setting],
+                                                                 data_process_rule_dict, document.doc_form,
+                                                                 'English', dataset_id)
                 except LLMBadRequestError:
                     raise ProviderNotInitializeError(
-                        f"No Embedding Model available. Please configure a valid provider "
-                        f"in the Settings -> Model Provider.")
+                        "No Embedding Model available. Please configure a valid provider "
+                        "in the Settings -> Model Provider.")
                 except ProviderTokenNotInitError as ex:
                     raise ProviderNotInitializeError(ex.description)
 
@@ -387,6 +409,7 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
         data_process_rule = documents[0].dataset_process_rule
         data_process_rule_dict = data_process_rule.to_dict()
         info_list = []
+        extract_settings = []
         for document in documents:
             if document.indexing_status in ['completed', 'error']:
                 raise DocumentAlreadyFinishedError()
@@ -409,42 +432,49 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
                 }
                 info_list.append(notion_info)
 
-        if dataset.data_source_type == 'upload_file':
-            file_details = db.session.query(UploadFile).filter(
-                UploadFile.tenant_id == current_user.current_tenant_id,
-                UploadFile.id.in_(info_list)
-            ).all()
+            if document.data_source_type == 'upload_file':
+                file_id = data_source_info['upload_file_id']
+                file_detail = db.session.query(UploadFile).filter(
+                    UploadFile.tenant_id == current_user.current_tenant_id,
+                    UploadFile.id == file_id
+                ).first()
 
-            if file_details is None:
-                raise NotFound("File not found.")
+                if file_detail is None:
+                    raise NotFound("File not found.")
 
+                extract_setting = ExtractSetting(
+                    datasource_type="upload_file",
+                    upload_file=file_detail,
+                    document_model=document.doc_form
+                )
+                extract_settings.append(extract_setting)
+
+            elif document.data_source_type == 'notion_import':
+                extract_setting = ExtractSetting(
+                    datasource_type="notion_import",
+                    notion_info={
+                        "notion_workspace_id": data_source_info['notion_workspace_id'],
+                        "notion_obj_id": data_source_info['notion_page_id'],
+                        "notion_page_type": data_source_info['type'],
+                        "tenant_id": current_user.current_tenant_id
+                    },
+                    document_model=document.doc_form
+                )
+                extract_settings.append(extract_setting)
+
+            else:
+                raise ValueError('Data source type not support')
             indexing_runner = IndexingRunner()
             try:
-                response = indexing_runner.file_indexing_estimate(current_user.current_tenant_id, file_details,
-                                                                  data_process_rule_dict, None,
-                                                                  'English', dataset_id)
+                response = indexing_runner.indexing_estimate(current_user.current_tenant_id, extract_settings,
+                                                             data_process_rule_dict, document.doc_form,
+                                                             'English', dataset_id)
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    f"No Embedding Model available. Please configure a valid provider "
-                    f"in the Settings -> Model Provider.")
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider.")
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
-        elif dataset.data_source_type == 'notion_import':
-
-            indexing_runner = IndexingRunner()
-            try:
-                response = indexing_runner.notion_indexing_estimate(current_user.current_tenant_id,
-                                                                    info_list,
-                                                                    data_process_rule_dict,
-                                                                    None, 'English', dataset_id)
-            except LLMBadRequestError:
-                raise ProviderNotInitializeError(
-                    f"No Embedding Model available. Please configure a valid provider "
-                    f"in the Settings -> Model Provider.")
-            except ProviderTokenNotInitError as ex:
-                raise ProviderNotInitializeError(ex.description)
-        else:
-            raise ValueError('Data source type not support')
         return response
 
 
@@ -599,7 +629,7 @@ class DocumentProcessingApi(DocumentResource):
         document = self.get_document(dataset_id, document_id)
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         if action == "pause":
@@ -663,7 +693,7 @@ class DocumentMetadataApi(DocumentResource):
         doc_metadata = req_data.get('doc_metadata')
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         if doc_type is None or doc_metadata is None:
@@ -710,7 +740,7 @@ class DocumentStatusApi(DocumentResource):
         document = self.get_document(dataset_id, document_id)
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         indexing_cache_key = 'document_{}_indexing'.format(document.id)

api/controllers/console/datasets/datasets_segments.py

@@ -1,30 +1,34 @@
-# -*- coding:utf-8 -*-
 import uuid
 from datetime import datetime
 
 import pandas as pd
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, marshal, reqparse
+from werkzeug.exceptions import Forbidden, NotFound
+
 import services
 from controllers.console import api
 from controllers.console.app.error import ProviderNotInitializeError
 from controllers.console.datasets.error import InvalidActionError, NoFileUploadedError, TooManyFilesError
 from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
+from controllers.console.wraps import (
+    account_initialization_required,
+    cloud_edition_billing_knowledge_limit_check,
+    cloud_edition_billing_resource_check,
+)
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from fields.segment_fields import segment_fields
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal, reqparse
 from libs.login import login_required
 from models.dataset import DocumentSegment
 from services.dataset_service import DatasetService, DocumentService, SegmentService
 from tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task
 from tasks.disable_segment_from_index_task import disable_segment_from_index_task
 from tasks.enable_segment_to_index_task import enable_segment_to_index_task
-from werkzeug.exceptions import Forbidden, NotFound
 
 
 class DatasetDocumentSegmentListApi(Resource):
@@ -123,7 +127,7 @@ class DatasetDocumentSegmentApi(Resource):
         # check user's model setting
         DatasetService.check_dataset_model_setting(dataset)
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         try:
@@ -142,8 +146,8 @@ class DatasetDocumentSegmentApi(Resource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    f"No Embedding Model available. Please configure a valid provider "
-                    f"in the Settings -> Model Provider.")
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider.")
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
 
@@ -207,6 +211,7 @@ class DatasetDocumentSegmentAddApi(Resource):
     @login_required
     @account_initialization_required
     @cloud_edition_billing_resource_check('vector_space')
+    @cloud_edition_billing_knowledge_limit_check('add_segment')
     def post(self, dataset_id, document_id):
         # check dataset
         dataset_id = str(dataset_id)
@@ -219,7 +224,7 @@ class DatasetDocumentSegmentAddApi(Resource):
         if not document:
             raise NotFound('Document not found.')
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
         # check embedding model setting
         if dataset.indexing_technique == 'high_quality':
@@ -233,8 +238,8 @@ class DatasetDocumentSegmentAddApi(Resource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    f"No Embedding Model available. Please configure a valid provider "
-                    f"in the Settings -> Model Provider.")
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider.")
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
         try:
@@ -285,8 +290,8 @@ class DatasetDocumentSegmentUpdateApi(Resource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    f"No Embedding Model available. Please configure a valid provider "
-                    f"in the Settings -> Model Provider.")
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider.")
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
             # check segment
@@ -298,7 +303,7 @@ class DatasetDocumentSegmentUpdateApi(Resource):
         if not segment:
             raise NotFound('Segment not found.')
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
         try:
             DatasetService.check_dataset_permission(dataset, current_user)
@@ -342,7 +347,7 @@ class DatasetDocumentSegmentUpdateApi(Resource):
         if not segment:
             raise NotFound('Segment not found.')
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
         try:
             DatasetService.check_dataset_permission(dataset, current_user)
@@ -357,6 +362,7 @@ class DatasetDocumentSegmentBatchImportApi(Resource):
     @login_required
     @account_initialization_required
     @cloud_edition_billing_resource_check('vector_space')
+    @cloud_edition_billing_knowledge_limit_check('add_segment')
     def post(self, dataset_id, document_id):
         # check dataset
         dataset_id = str(dataset_id)

api/controllers/console/datasets/file.py

@@ -1,15 +1,20 @@
-import services
-from controllers.console import api
-from controllers.console.datasets.error import (FileTooLargeError, NoFileUploadedError, TooManyFilesError,
-                                                UnsupportedFileTypeError)
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
-from fields.file_fields import file_fields, upload_config_fields
 from flask import current_app, request
 from flask_login import current_user
 from flask_restful import Resource, marshal_with
+
+import services
+from controllers.console import api
+from controllers.console.datasets.error import (
+    FileTooLargeError,
+    NoFileUploadedError,
+    TooManyFilesError,
+    UnsupportedFileTypeError,
+)
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
+from fields.file_fields import file_fields, upload_config_fields
 from libs.login import login_required
-from services.file_service import FileService
+from services.file_service import ALLOWED_EXTENSIONS, UNSTRUSTURED_ALLOWED_EXTENSIONS, FileService
 
 PREVIEW_WORDS_LIMIT = 3000
 
@@ -34,6 +39,7 @@ class FileApi(Resource):
     @login_required
     @account_initialization_required
     @marshal_with(file_fields)
+    @cloud_edition_billing_resource_check(resource='documents')
     def post(self):
 
         # get file from request
@@ -71,11 +77,7 @@ class FileSupportTypeApi(Resource):
     @account_initialization_required
     def get(self):
         etl_type = current_app.config['ETL_TYPE']
-        if etl_type == 'Unstructured':
-            allowed_extensions = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm', 'xlsx',
-                                  'docx', 'csv', 'eml', 'msg', 'pptx', 'ppt', 'xml']
-        else:
-            allowed_extensions = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm', 'xlsx', 'docx', 'csv']
+        allowed_extensions = UNSTRUSTURED_ALLOWED_EXTENSIONS if etl_type == 'Unstructured' else ALLOWED_EXTENSIONS
         return {'allowed_extensions': allowed_extensions}
 
 

api/controllers/console/datasets/hit_testing.py

@@ -1,22 +1,31 @@
 import logging
 
+from flask_login import current_user
+from flask_restful import Resource, marshal, reqparse
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
+
 import services
 from controllers.console import api
-from controllers.console.app.error import (CompletionRequestError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.console.datasets.error import DatasetNotInitializedError, HighQualityDatasetOnlyError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
-from core.errors.error import (LLMBadRequestError, ModelCurrentlyNotSupportError, ProviderTokenNotInitError,
-                               QuotaExceededError)
+from core.errors.error import (
+    LLMBadRequestError,
+    ModelCurrentlyNotSupportError,
+    ProviderTokenNotInitError,
+    QuotaExceededError,
+)
 from core.model_runtime.errors.invoke import InvokeError
 from fields.hit_testing_fields import hit_testing_record_fields
-from flask_login import current_user
-from flask_restful import Resource, marshal, reqparse
 from libs.login import login_required
 from services.dataset_service import DatasetService
 from services.hit_testing_service import HitTestingService
-from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 
 class HitTestingApi(Resource):
@@ -67,8 +76,8 @@ class HitTestingApi(Resource):
             raise ProviderModelCurrentlyNotSupportError()
         except LLMBadRequestError:
             raise ProviderNotInitializeError(
-                f"No Embedding Model or Reranking Model available. Please configure a valid provider "
-                f"in the Settings -> Model Provider.")
+                "No Embedding Model or Reranking Model available. Please configure a valid provider "
+                "in the Settings -> Model Provider.")
         except InvokeError as e:
             raise CompletionRequestError(e.description)
         except ValueError as e:

api/controllers/console/error.py

@@ -13,6 +13,16 @@ class NotSetupError(BaseHTTPException):
                   "Please proceed with the initialization and installation process first."
     code = 401
 
+class NotInitValidateError(BaseHTTPException):
+    error_code = 'not_init_validated'
+    description = "Init validation has not been completed yet. " \
+                  "Please proceed with the init validation process first."
+    code = 401
+
+class InitValidateFailedError(BaseHTTPException):
+    error_code = 'init_validate_failed'
+    description = "Init validation failed. Please check the password and try again."
+    code = 401
 
 class AccountNotLinkTenantError(BaseHTTPException):
     error_code = 'account_not_link_tenant'

api/controllers/console/explore/audio.py

@@ -1,37 +1,44 @@
-# -*- coding:utf-8 -*-
 import logging
 
+from flask import request
+from werkzeug.exceptions import InternalServerError
+
 import services
 from controllers.console import api
-from controllers.console.app.error import (AppUnavailableError, AudioTooLargeError, CompletionRequestError,
-                                           NoAudioUploadedError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderNotSupportSpeechToTextError,
-                                           ProviderQuotaExceededError, UnsupportedAudioTypeError)
+from controllers.console.app.error import (
+    AppUnavailableError,
+    AudioTooLargeError,
+    CompletionRequestError,
+    NoAudioUploadedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderNotSupportSpeechToTextError,
+    ProviderQuotaExceededError,
+    UnsupportedAudioTypeError,
+)
 from controllers.console.explore.wraps import InstalledAppResource
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import request
-from models.model import AppModelConfig
 from services.audio_service import AudioService
-from services.errors.audio import (AudioTooLargeServiceError, NoAudioUploadedServiceError,
-                                   ProviderNotSupportSpeechToTextServiceError, UnsupportedAudioTypeServiceError)
-from werkzeug.exceptions import InternalServerError
+from services.errors.audio import (
+    AudioTooLargeServiceError,
+    NoAudioUploadedServiceError,
+    ProviderNotSupportSpeechToTextServiceError,
+    UnsupportedAudioTypeServiceError,
+)
 
 
 class ChatAudioApi(InstalledAppResource):
     def post(self, installed_app):
         app_model = installed_app.app
-        app_model_config: AppModelConfig = app_model.app_model_config
-
-        if not app_model_config.speech_to_text_dict['enabled']:
-            raise AppUnavailableError()
 
         file = request.files['file']
 
         try:
-            response = AudioService.transcript(
-                tenant_id=app_model.tenant_id,
+            response = AudioService.transcript_asr(
+                app_model=app_model,
                 file=file,
+                end_user=None
             )
 
             return response
@@ -59,6 +66,45 @@ class ChatAudioApi(InstalledAppResource):
         except Exception as e:
             logging.exception("internal server error.")
             raise InternalServerError()
-        
 
-api.add_resource(ChatAudioApi, '/installed-apps/<uuid:installed_app_id>/audio-to-text', endpoint='installed_app_audio')
+
+class ChatTextApi(InstalledAppResource):
+    def post(self, installed_app):
+        app_model = installed_app.app
+
+        try:
+            response = AudioService.transcript_tts(
+                app_model=app_model,
+                text=request.form['text'],
+                voice=request.form.get('voice'),
+                streaming=False
+            )
+            return {'data': response.data.decode('latin1')}
+        except services.errors.app_model_config.AppModelConfigBrokenError:
+            logging.exception("App model config broken.")
+            raise AppUnavailableError()
+        except NoAudioUploadedServiceError:
+            raise NoAudioUploadedError()
+        except AudioTooLargeServiceError as e:
+            raise AudioTooLargeError(str(e))
+        except UnsupportedAudioTypeServiceError:
+            raise UnsupportedAudioTypeError()
+        except ProviderNotSupportSpeechToTextServiceError:
+            raise ProviderNotSupportSpeechToTextError()
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except InvokeError as e:
+            raise CompletionRequestError(e.description)
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logging.exception("internal server error.")
+            raise InternalServerError()
+
+
+api.add_resource(ChatAudioApi, '/installed-apps/<uuid:installed_app_id>/audio-to-text', endpoint='installed_app_audio')
+api.add_resource(ChatTextApi, '/installed-apps/<uuid:installed_app_id>/text-to-audio', endpoint='installed_app_text')

api/controllers/console/explore/completion.py

@@ -1,27 +1,31 @@
-# -*- coding:utf-8 -*-
-import json
 import logging
 from datetime import datetime
-from typing import Generator, Union
+
+from flask_login import current_user
+from flask_restful import reqparse
+from werkzeug.exceptions import InternalServerError, NotFound
 
 import services
 from controllers.console import api
-from controllers.console.app.error import (AppUnavailableError, CompletionRequestError, ConversationCompletedError,
-                                           ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    AppUnavailableError,
+    CompletionRequestError,
+    ConversationCompletedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.console.explore.error import NotChatAppError, NotCompletionAppError
 from controllers.console.explore.wraps import InstalledAppResource
-from core.application_queue_manager import ApplicationQueueManager
-from core.entities.application_entities import InvokeFrom
+from core.app.apps.base_app_queue_manager import AppQueueManager
+from core.app.entities.app_invoke_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
 from extensions.ext_database import db
-from flask import Response, stream_with_context
-from flask_login import current_user
-from flask_restful import reqparse
+from libs import helper
 from libs.helper import uuid_value
-from services.completion_service import CompletionService
-from werkzeug.exceptions import InternalServerError, NotFound
+from models.model import AppMode
+from services.app_generate_service import AppGenerateService
 
 
 # define completion api for user
@@ -47,7 +51,7 @@ class CompletionApi(InstalledAppResource):
         db.session.commit()
 
         try:
-            response = CompletionService.completion(
+            response = AppGenerateService.generate(
                 app_model=app_model,
                 user=current_user,
                 args=args,
@@ -55,7 +59,7 @@ class CompletionApi(InstalledAppResource):
                 streaming=streaming
             )
 
-            return compact_response(response)
+            return helper.compact_generate_response(response)
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
         except services.errors.conversation.ConversationCompletedError:
@@ -84,7 +88,7 @@ class CompletionStopApi(InstalledAppResource):
         if app_model.mode != 'completion':
             raise NotCompletionAppError()
 
-        ApplicationQueueManager.set_stop_flag(task_id, InvokeFrom.EXPLORE, current_user.id)
+        AppQueueManager.set_stop_flag(task_id, InvokeFrom.EXPLORE, current_user.id)
 
         return {'result': 'success'}, 200
 
@@ -92,34 +96,33 @@ class CompletionStopApi(InstalledAppResource):
 class ChatApi(InstalledAppResource):
     def post(self, installed_app):
         app_model = installed_app.app
-        if app_model.mode != 'chat':
+        app_mode = AppMode.value_of(app_model.mode)
+        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, required=True, location='json')
         parser.add_argument('query', type=str, required=True, location='json')
         parser.add_argument('files', type=list, required=False, location='json')
-        parser.add_argument('response_mode', type=str, choices=['blocking', 'streaming'], location='json')
         parser.add_argument('conversation_id', type=uuid_value, location='json')
         parser.add_argument('retriever_from', type=str, required=False, default='explore_app', location='json')
         args = parser.parse_args()
 
-        streaming = args['response_mode'] == 'streaming'
         args['auto_generate_name'] = False
 
         installed_app.last_used_at = datetime.utcnow()
         db.session.commit()
 
         try:
-            response = CompletionService.completion(
+            response = AppGenerateService.generate(
                 app_model=app_model,
                 user=current_user,
                 args=args,
                 invoke_from=InvokeFrom.EXPLORE,
-                streaming=streaming
+                streaming=True
             )
 
-            return compact_response(response)
+            return helper.compact_generate_response(response)
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
         except services.errors.conversation.ConversationCompletedError:
@@ -145,47 +148,15 @@ class ChatApi(InstalledAppResource):
 class ChatStopApi(InstalledAppResource):
     def post(self, installed_app, task_id):
         app_model = installed_app.app
-        if app_model.mode != 'chat':
+        app_mode = AppMode.value_of(app_model.mode)
+        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
             raise NotChatAppError()
 
-        ApplicationQueueManager.set_stop_flag(task_id, InvokeFrom.EXPLORE, current_user.id)
+        AppQueueManager.set_stop_flag(task_id, InvokeFrom.EXPLORE, current_user.id)
 
         return {'result': 'success'}, 200
 
 
-def compact_response(response: Union[dict, Generator]) -> Response:
-    if isinstance(response, dict):
-        return Response(response=json.dumps(response), status=200, mimetype='application/json')
-    else:
-        def generate() -> Generator:
-            try:
-                for chunk in response:
-                    yield chunk
-            except services.errors.conversation.ConversationNotExistsError:
-                yield "data: " + json.dumps(api.handle_error(NotFound("Conversation Not Exists.")).get_json()) + "\n\n"
-            except services.errors.conversation.ConversationCompletedError:
-                yield "data: " + json.dumps(api.handle_error(ConversationCompletedError()).get_json()) + "\n\n"
-            except services.errors.app_model_config.AppModelConfigBrokenError:
-                logging.exception("App model config broken.")
-                yield "data: " + json.dumps(api.handle_error(AppUnavailableError()).get_json()) + "\n\n"
-            except ProviderTokenNotInitError as ex:
-                yield "data: " + json.dumps(api.handle_error(ProviderNotInitializeError(ex.description)).get_json()) + "\n\n"
-            except QuotaExceededError:
-                yield "data: " + json.dumps(api.handle_error(ProviderQuotaExceededError()).get_json()) + "\n\n"
-            except ModelCurrentlyNotSupportError:
-                yield "data: " + json.dumps(api.handle_error(ProviderModelCurrentlyNotSupportError()).get_json()) + "\n\n"
-            except InvokeError as e:
-                yield "data: " + json.dumps(api.handle_error(CompletionRequestError(e.description)).get_json()) + "\n\n"
-            except ValueError as e:
-                yield "data: " + json.dumps(api.handle_error(e).get_json()) + "\n\n"
-            except Exception:
-                logging.exception("internal server error.")
-                yield "data: " + json.dumps(api.handle_error(InternalServerError()).get_json()) + "\n\n"
-
-        return Response(stream_with_context(generate()), status=200,
-                        mimetype='text/event-stream')
-
-
 api.add_resource(CompletionApi, '/installed-apps/<uuid:installed_app_id>/completion-messages', endpoint='installed_app_completion')
 api.add_resource(CompletionStopApi, '/installed-apps/<uuid:installed_app_id>/completion-messages/<string:task_id>/stop', endpoint='installed_app_stop_completion')
 api.add_resource(ChatApi, '/installed-apps/<uuid:installed_app_id>/chat-messages', endpoint='installed_app_chat_completion')

api/controllers/console/explore/conversation.py

@@ -1,16 +1,17 @@
-# -*- coding:utf-8 -*-
+from flask_login import current_user
+from flask_restful import marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import NotFound
+
 from controllers.console import api
 from controllers.console.explore.error import NotChatAppError
 from controllers.console.explore.wraps import InstalledAppResource
 from fields.conversation_fields import conversation_infinite_scroll_pagination_fields, simple_conversation_fields
-from flask_login import current_user
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
-from libs.helper import TimestampField, uuid_value
+from libs.helper import uuid_value
+from models.model import AppMode
 from services.conversation_service import ConversationService
 from services.errors.conversation import ConversationNotExistsError, LastConversationNotExistsError
 from services.web_conversation_service import WebConversationService
-from werkzeug.exceptions import NotFound
 
 
 class ConversationListApi(InstalledAppResource):
@@ -18,7 +19,8 @@ class ConversationListApi(InstalledAppResource):
     @marshal_with(conversation_infinite_scroll_pagination_fields)
     def get(self, installed_app):
         app_model = installed_app.app
-        if app_model.mode != 'chat':
+        app_mode = AppMode.value_of(app_model.mode)
+        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -47,7 +49,8 @@ class ConversationListApi(InstalledAppResource):
 class ConversationApi(InstalledAppResource):
     def delete(self, installed_app, c_id):
         app_model = installed_app.app
-        if app_model.mode != 'chat':
+        app_mode = AppMode.value_of(app_model.mode)
+        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -65,7 +68,8 @@ class ConversationRenameApi(InstalledAppResource):
     @marshal_with(simple_conversation_fields)
     def post(self, installed_app, c_id):
         app_model = installed_app.app
-        if app_model.mode != 'chat':
+        app_mode = AppMode.value_of(app_model.mode)
+        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -91,7 +95,8 @@ class ConversationPinApi(InstalledAppResource):
 
     def patch(self, installed_app, c_id):
         app_model = installed_app.app
-        if app_model.mode != 'chat':
+        app_mode = AppMode.value_of(app_model.mode)
+        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -107,7 +112,8 @@ class ConversationPinApi(InstalledAppResource):
 class ConversationUnPinApi(InstalledAppResource):
     def patch(self, installed_app, c_id):
         app_model = installed_app.app
-        if app_model.mode != 'chat':
+        app_mode = AppMode.value_of(app_model.mode)
+        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
             raise NotChatAppError()
 
         conversation_id = str(c_id)

api/controllers/console/explore/error.py

@@ -1,4 +1,3 @@
-# -*- coding:utf-8 -*-
 from libs.exception import BaseHTTPException
 
 
@@ -10,7 +9,13 @@ class NotCompletionAppError(BaseHTTPException):
 
 class NotChatAppError(BaseHTTPException):
     error_code = 'not_chat_app'
-    description = "Not Chat App"
+    description = "App mode is invalid."
+    code = 400
+
+
+class NotWorkflowAppError(BaseHTTPException):
+    error_code = 'not_workflow_app'
+    description = "Only support workflow app."
     code = 400
 
 

api/controllers/console/explore/installed_app.py

@@ -1,18 +1,18 @@
-# -*- coding:utf-8 -*-
 from datetime import datetime
 
+from flask_login import current_user
+from flask_restful import Resource, inputs, marshal_with, reqparse
+from sqlalchemy import and_
+from werkzeug.exceptions import BadRequest, Forbidden, NotFound
+
 from controllers.console import api
 from controllers.console.explore.wraps import InstalledAppResource
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_database import db
 from fields.installed_app_fields import installed_app_list_fields
-from flask_login import current_user
-from flask_restful import Resource, inputs, marshal_with, reqparse
 from libs.login import login_required
 from models.model import App, InstalledApp, RecommendedApp
 from services.account_service import TenantService
-from sqlalchemy import and_
-from werkzeug.exceptions import BadRequest, Forbidden, NotFound
 
 
 class InstalledAppsListApi(Resource):
@@ -33,8 +33,8 @@ class InstalledAppsListApi(Resource):
                 'app_owner_tenant_id': installed_app.app_owner_tenant_id,
                 'is_pinned': installed_app.is_pinned,
                 'last_used_at': installed_app.last_used_at,
-                "editable": current_user.role in ["owner", "admin"],
-                "uninstallable": current_tenant_id == installed_app.app_owner_tenant_id
+                'editable': current_user.role in ["owner", "admin"],
+                'uninstallable': current_tenant_id == installed_app.app_owner_tenant_id
             }
             for installed_app in installed_apps
         ]

api/controllers/console/explore/message.py

@@ -1,40 +1,46 @@
-# -*- coding:utf-8 -*-
-import json
 import logging
-from typing import Generator, Union
 
-import services
-from controllers.console import api
-from controllers.console.app.error import (AppMoreLikeThisDisabledError, CompletionRequestError,
-                                           ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
-from controllers.console.explore.error import (AppSuggestedQuestionsAfterAnswerDisabledError, NotChatAppError,
-                                               NotCompletionAppError)
-from controllers.console.explore.wraps import InstalledAppResource
-from core.entities.application_entities import InvokeFrom
-from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
-from core.model_runtime.errors.invoke import InvokeError
-from fields.message_fields import message_infinite_scroll_pagination_fields
-from flask import Response, stream_with_context
 from flask_login import current_user
 from flask_restful import marshal_with, reqparse
 from flask_restful.inputs import int_range
+from werkzeug.exceptions import InternalServerError, NotFound
+
+import services
+from controllers.console import api
+from controllers.console.app.error import (
+    AppMoreLikeThisDisabledError,
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
+from controllers.console.explore.error import (
+    AppSuggestedQuestionsAfterAnswerDisabledError,
+    NotChatAppError,
+    NotCompletionAppError,
+)
+from controllers.console.explore.wraps import InstalledAppResource
+from core.app.entities.app_invoke_entities import InvokeFrom
+from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
+from core.model_runtime.errors.invoke import InvokeError
+from fields.message_fields import message_infinite_scroll_pagination_fields
+from libs import helper
 from libs.helper import uuid_value
-from services.completion_service import CompletionService
+from models.model import AppMode
+from services.app_generate_service import AppGenerateService
 from services.errors.app import MoreLikeThisDisabledError
 from services.errors.conversation import ConversationNotExistsError
 from services.errors.message import MessageNotExistsError, SuggestedQuestionsAfterAnswerDisabledError
 from services.message_service import MessageService
-from werkzeug.exceptions import InternalServerError, NotFound
 
 
 class MessageListApi(InstalledAppResource):
-
     @marshal_with(message_infinite_scroll_pagination_fields)
     def get(self, installed_app):
         app_model = installed_app.app
 
-        if app_model.mode != 'chat':
+        app_mode = AppMode.value_of(app_model.mode)
+        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -51,7 +57,6 @@ class MessageListApi(InstalledAppResource):
         except services.errors.message.FirstMessageNotExistsError:
             raise NotFound("First Message Not Exists.")
 
-
 class MessageFeedbackApi(InstalledAppResource):
     def post(self, installed_app, message_id):
         app_model = installed_app.app
@@ -85,14 +90,14 @@ class MessageMoreLikeThisApi(InstalledAppResource):
         streaming = args['response_mode'] == 'streaming'
 
         try:
-            response = CompletionService.generate_more_like_this(
+            response = AppGenerateService.generate_more_like_this(
                 app_model=app_model,
                 user=current_user,
                 message_id=message_id,
                 invoke_from=InvokeFrom.EXPLORE,
                 streaming=streaming
             )
-            return compact_response(response)
+            return helper.compact_generate_response(response)
         except MessageNotExistsError:
             raise NotFound("Message Not Exists.")
         except MoreLikeThisDisabledError:
@@ -112,41 +117,12 @@ class MessageMoreLikeThisApi(InstalledAppResource):
             raise InternalServerError()
 
 
-def compact_response(response: Union[dict, Generator]) -> Response:
-    if isinstance(response, dict):
-        return Response(response=json.dumps(response), status=200, mimetype='application/json')
-    else:
-        def generate() -> Generator:
-            try:
-                for chunk in response:
-                    yield chunk
-            except MessageNotExistsError:
-                yield "data: " + json.dumps(api.handle_error(NotFound("Message Not Exists.")).get_json()) + "\n\n"
-            except MoreLikeThisDisabledError:
-                yield "data: " + json.dumps(api.handle_error(AppMoreLikeThisDisabledError()).get_json()) + "\n\n"
-            except ProviderTokenNotInitError as ex:
-                yield "data: " + json.dumps(api.handle_error(ProviderNotInitializeError(ex.description)).get_json()) + "\n\n"
-            except QuotaExceededError:
-                yield "data: " + json.dumps(api.handle_error(ProviderQuotaExceededError()).get_json()) + "\n\n"
-            except ModelCurrentlyNotSupportError:
-                yield "data: " + json.dumps(api.handle_error(ProviderModelCurrentlyNotSupportError()).get_json()) + "\n\n"
-            except InvokeError as e:
-                yield "data: " + json.dumps(api.handle_error(CompletionRequestError(e.description)).get_json()) + "\n\n"
-            except ValueError as e:
-                yield "data: " + json.dumps(api.handle_error(e).get_json()) + "\n\n"
-            except Exception:
-                logging.exception("internal server error.")
-                yield "data: " + json.dumps(api.handle_error(InternalServerError()).get_json()) + "\n\n"
-
-        return Response(stream_with_context(generate()), status=200,
-                        mimetype='text/event-stream')
-
-
 class MessageSuggestedQuestionApi(InstalledAppResource):
     def get(self, installed_app, message_id):
         app_model = installed_app.app
-        if app_model.mode != 'chat':
-            raise NotCompletionAppError()
+        app_mode = AppMode.value_of(app_model.mode)
+        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+            raise NotChatAppError()
 
         message_id = str(message_id)
 
@@ -154,7 +130,8 @@ class MessageSuggestedQuestionApi(InstalledAppResource):
             questions = MessageService.get_suggested_questions_after_answer(
                 app_model=app_model,
                 user=current_user,
-                message_id=message_id
+                message_id=message_id,
+                invoke_from=InvokeFrom.EXPLORE
             )
         except MessageNotExistsError:
             raise NotFound("Message not found")

api/controllers/console/explore/parameter.py

@@ -1,9 +1,12 @@
-# -*- coding:utf-8 -*-
-from controllers.console import api
-from controllers.console.explore.wraps import InstalledAppResource
+
 from flask import current_app
 from flask_restful import fields, marshal_with
-from models.model import InstalledApp
+
+from controllers.console import api
+from controllers.console.app.error import AppUnavailableError
+from controllers.console.explore.wraps import InstalledAppResource
+from models.model import AppMode, InstalledApp
+from services.app_service import AppService
 
 
 class AppParameterApi(InstalledAppResource):
@@ -27,6 +30,7 @@ class AppParameterApi(InstalledAppResource):
         'suggested_questions': fields.Raw,
         'suggested_questions_after_answer': fields.Raw,
         'speech_to_text': fields.Raw,
+        'text_to_speech': fields.Raw,
         'retriever_resource': fields.Raw,
         'annotation_reply': fields.Raw,
         'more_like_this': fields.Raw,
@@ -40,23 +44,52 @@ class AppParameterApi(InstalledAppResource):
     def get(self, installed_app: InstalledApp):
         """Retrieve app parameters."""
         app_model = installed_app.app
-        app_model_config = app_model.app_model_config
+
+        if app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]:
+            workflow = app_model.workflow
+            if workflow is None:
+                raise AppUnavailableError()
+
+            features_dict = workflow.features_dict
+            user_input_form = workflow.user_input_form(to_old_structure=True)
+        else:
+            app_model_config = app_model.app_model_config
+            features_dict = app_model_config.to_dict()
+
+            user_input_form = features_dict.get('user_input_form', [])
 
         return {
-            'opening_statement': app_model_config.opening_statement,
-            'suggested_questions': app_model_config.suggested_questions_list,
-            'suggested_questions_after_answer': app_model_config.suggested_questions_after_answer_dict,
-            'speech_to_text': app_model_config.speech_to_text_dict,
-            'retriever_resource': app_model_config.retriever_resource_dict,
-            'annotation_reply': app_model_config.annotation_reply_dict,
-            'more_like_this': app_model_config.more_like_this_dict,
-            'user_input_form': app_model_config.user_input_form_list,
-            'sensitive_word_avoidance': app_model_config.sensitive_word_avoidance_dict,
-            'file_upload': app_model_config.file_upload_dict,
+            'opening_statement': features_dict.get('opening_statement'),
+            'suggested_questions': features_dict.get('suggested_questions', []),
+            'suggested_questions_after_answer': features_dict.get('suggested_questions_after_answer',
+                                                                  {"enabled": False}),
+            'speech_to_text': features_dict.get('speech_to_text', {"enabled": False}),
+            'text_to_speech': features_dict.get('text_to_speech', {"enabled": False}),
+            'retriever_resource': features_dict.get('retriever_resource', {"enabled": False}),
+            'annotation_reply': features_dict.get('annotation_reply', {"enabled": False}),
+            'more_like_this': features_dict.get('more_like_this', {"enabled": False}),
+            'user_input_form': user_input_form,
+            'sensitive_word_avoidance': features_dict.get('sensitive_word_avoidance',
+                                                          {"enabled": False, "type": "", "configs": []}),
+            'file_upload': features_dict.get('file_upload', {"image": {
+                                                     "enabled": False,
+                                                     "number_limits": 3,
+                                                     "detail": "high",
+                                                     "transfer_methods": ["remote_url", "local_file"]
+                                                 }}),
             'system_parameters': {
                 'image_file_size_limit': current_app.config.get('UPLOAD_IMAGE_FILE_SIZE_LIMIT')
             }
         }
 
 
-api.add_resource(AppParameterApi, '/installed-apps/<uuid:installed_app_id>/parameters', endpoint='installed_app_parameters')
+class ExploreAppMetaApi(InstalledAppResource):
+    def get(self, installed_app: InstalledApp):
+        """Get app meta"""
+        app_model = installed_app.app
+        return AppService().get_app_meta(app_model)
+
+
+api.add_resource(AppParameterApi, '/installed-apps/<uuid:installed_app_id>/parameters',
+                 endpoint='installed_app_parameters')
+api.add_resource(ExploreAppMetaApi, '/installed-apps/<uuid:installed_app_id>/meta', endpoint='installed_app_meta')

api/controllers/console/explore/recommended_app.py

@@ -1,14 +1,11 @@
-# -*- coding:utf-8 -*-
-from controllers.console import api
-from controllers.console.app.error import AppNotFoundError
-from controllers.console.wraps import account_initialization_required
-from extensions.ext_database import db
 from flask_login import current_user
-from flask_restful import Resource, fields, marshal_with
+from flask_restful import Resource, fields, marshal_with, reqparse
+
+from constants.languages import languages
+from controllers.console import api
+from controllers.console.wraps import account_initialization_required
 from libs.login import login_required
-from models.model import App, InstalledApp, RecommendedApp
-from services.account_service import TenantService
-from sqlalchemy import and_
+from services.recommended_app_service import RecommendedAppService
 
 app_fields = {
     'id': fields.String,
@@ -26,10 +23,7 @@ recommended_app_fields = {
     'privacy_policy': fields.String,
     'category': fields.String,
     'position': fields.Integer,
-    'is_listed': fields.Boolean,
-    'install_count': fields.Integer,
-    'installed': fields.Boolean,
-    'editable': fields.Boolean
+    'is_listed': fields.Boolean
 }
 
 recommended_app_list_fields = {
@@ -43,95 +37,27 @@ class RecommendedAppListApi(Resource):
     @account_initialization_required
     @marshal_with(recommended_app_list_fields)
     def get(self):
-        language_prefix = current_user.interface_language if current_user.interface_language else 'en-US'
+        # language args
+        parser = reqparse.RequestParser()
+        parser.add_argument('language', type=str, location='args')
+        args = parser.parse_args()
 
-        recommended_apps = db.session.query(RecommendedApp).filter(
-            RecommendedApp.is_listed == True,
-            RecommendedApp.language == language_prefix
-        ).all()
+        if args.get('language') and args.get('language') in languages:
+            language_prefix = args.get('language')
+        elif current_user and current_user.interface_language:
+            language_prefix = current_user.interface_language
+        else:
+            language_prefix = languages[0]
 
-        categories = set()
-        current_user.role = TenantService.get_user_role(current_user, current_user.current_tenant)
-        recommended_apps_result = []
-        for recommended_app in recommended_apps:
-            installed = db.session.query(InstalledApp).filter(
-                and_(
-                    InstalledApp.app_id == recommended_app.app_id,
-                    InstalledApp.tenant_id == current_user.current_tenant_id
-                )
-            ).first() is not None
-
-            app = recommended_app.app
-            if not app or not app.is_public:
-                continue
-
-            site = app.site
-            if not site:
-                continue
-
-            recommended_app_result = {
-                'id': recommended_app.id,
-                'app': app,
-                'app_id': recommended_app.app_id,
-                'description': site.description,
-                'copyright': site.copyright,
-                'privacy_policy': site.privacy_policy,
-                'category': recommended_app.category,
-                'position': recommended_app.position,
-                'is_listed': recommended_app.is_listed,
-                'install_count': recommended_app.install_count,
-                'installed': installed,
-                'editable': current_user.role in ['owner', 'admin'],
-            }
-            recommended_apps_result.append(recommended_app_result)
-
-            categories.add(recommended_app.category)  # add category to categories
-
-        return {'recommended_apps': recommended_apps_result, 'categories': list(categories)}
+        return RecommendedAppService.get_recommended_apps_and_categories(language_prefix)
 
 
 class RecommendedAppApi(Resource):
-    model_config_fields = {
-        'opening_statement': fields.String,
-        'suggested_questions': fields.Raw(attribute='suggested_questions_list'),
-        'suggested_questions_after_answer': fields.Raw(attribute='suggested_questions_after_answer_dict'),
-        'more_like_this': fields.Raw(attribute='more_like_this_dict'),
-        'model': fields.Raw(attribute='model_dict'),
-        'user_input_form': fields.Raw(attribute='user_input_form_list'),
-        'pre_prompt': fields.String,
-        'agent_mode': fields.Raw(attribute='agent_mode_dict'),
-    }
-
-    app_simple_detail_fields = {
-        'id': fields.String,
-        'name': fields.String,
-        'icon': fields.String,
-        'icon_background': fields.String,
-        'mode': fields.String,
-        'app_model_config': fields.Nested(model_config_fields),
-    }
-
     @login_required
     @account_initialization_required
-    @marshal_with(app_simple_detail_fields)
     def get(self, app_id):
         app_id = str(app_id)
-
-        # is in public recommended list
-        recommended_app = db.session.query(RecommendedApp).filter(
-            RecommendedApp.is_listed == True,
-            RecommendedApp.app_id == app_id
-        ).first()
-
-        if not recommended_app:
-            raise AppNotFoundError
-
-        # get app detail
-        app = db.session.query(App).filter(App.id == app_id).first()
-        if not app or not app.is_public:
-            raise AppNotFoundError
-
-        return app
+        return RecommendedAppService.get_recommend_app_detail(app_id)
 
 
 api.add_resource(RecommendedAppListApi, '/explore/apps')

api/controllers/console/explore/saved_message.py

@@ -1,14 +1,15 @@
+from flask_login import current_user
+from flask_restful import fields, marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import NotFound
+
 from controllers.console import api
 from controllers.console.explore.error import NotCompletionAppError
 from controllers.console.explore.wraps import InstalledAppResource
 from fields.conversation_fields import message_file_fields
-from flask_login import current_user
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
 from libs.helper import TimestampField, uuid_value
 from services.errors.message import MessageNotExistsError
 from services.saved_message_service import SavedMessageService
-from werkzeug.exceptions import NotFound
 
 feedback_fields = {
     'rating': fields.String

api/controllers/console/explore/workflow.py

@@ -0,0 +1,85 @@
+import logging
+
+from flask_restful import reqparse
+from werkzeug.exceptions import InternalServerError
+
+from controllers.console import api
+from controllers.console.app.error import (
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
+from controllers.console.explore.error import NotWorkflowAppError
+from controllers.console.explore.wraps import InstalledAppResource
+from core.app.apps.base_app_queue_manager import AppQueueManager
+from core.app.entities.app_invoke_entities import InvokeFrom
+from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
+from core.model_runtime.errors.invoke import InvokeError
+from libs import helper
+from libs.login import current_user
+from models.model import AppMode, InstalledApp
+from services.app_generate_service import AppGenerateService
+
+logger = logging.getLogger(__name__)
+
+
+class InstalledAppWorkflowRunApi(InstalledAppResource):
+    def post(self, installed_app: InstalledApp):
+        """
+        Run workflow
+        """
+        app_model = installed_app.app
+        app_mode = AppMode.value_of(app_model.mode)
+        if app_mode != AppMode.WORKFLOW:
+            raise NotWorkflowAppError()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('inputs', type=dict, required=True, nullable=False, location='json')
+        parser.add_argument('files', type=list, required=False, location='json')
+        args = parser.parse_args()
+
+        try:
+            response = AppGenerateService.generate(
+                app_model=app_model,
+                user=current_user,
+                args=args,
+                invoke_from=InvokeFrom.EXPLORE,
+                streaming=True
+            )
+
+            return helper.compact_generate_response(response)
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except InvokeError as e:
+            raise CompletionRequestError(e.description)
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logging.exception("internal server error.")
+            raise InternalServerError()
+
+
+class InstalledAppWorkflowTaskStopApi(InstalledAppResource):
+    def post(self, installed_app: InstalledApp, task_id: str):
+        """
+        Stop workflow task
+        """
+        app_model = installed_app.app
+        app_mode = AppMode.value_of(app_model.mode)
+        if app_mode != AppMode.WORKFLOW:
+            raise NotWorkflowAppError()
+
+        AppQueueManager.set_stop_flag(task_id, InvokeFrom.EXPLORE, current_user.id)
+
+        return {
+            "result": "success"
+        }
+
+
+api.add_resource(InstalledAppWorkflowRunApi, '/installed-apps/<uuid:installed_app_id>/workflows/run')
+api.add_resource(InstalledAppWorkflowTaskStopApi, '/installed-apps/<uuid:installed_app_id>/workflows/tasks/<string:task_id>/stop')

api/controllers/console/explore/wraps.py

@@ -1,12 +1,13 @@
 from functools import wraps
 
+from flask_login import current_user
+from flask_restful import Resource
+from werkzeug.exceptions import NotFound
+
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
-from flask_login import current_user
-from flask_restful import Resource
 from libs.login import login_required
 from models.model import InstalledApp
-from werkzeug.exceptions import NotFound
 
 
 def installed_app_required(view=None):

api/controllers/console/extension.py

@@ -1,9 +1,10 @@
+from flask_login import current_user
+from flask_restful import Resource, marshal_with, reqparse
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from fields.api_based_extension_fields import api_based_extension_fields
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
 from libs.login import login_required
 from models.api_based_extension import APIBasedExtension
 from services.api_based_extension_service import APIBasedExtensionService

api/controllers/console/feature.py

@@ -1,12 +1,15 @@
 from flask_login import current_user
 from flask_restful import Resource
+
 from services.feature_service import FeatureService
 
 from . import api
+from .wraps import cloud_utm_record
 
 
 class FeatureApi(Resource):
 
+    @cloud_utm_record
     def get(self):
         return FeatureService.get_features(current_user.current_tenant_id).dict()
 

api/controllers/console/init_validate.py

@@ -0,0 +1,49 @@
+import os
+
+from flask import current_app, session
+from flask_restful import Resource, reqparse
+
+from libs.helper import str_len
+from models.model import DifySetup
+from services.account_service import TenantService
+
+from . import api
+from .error import AlreadySetupError, InitValidateFailedError
+from .wraps import only_edition_self_hosted
+
+
+class InitValidateAPI(Resource):
+
+    def get(self):
+        init_status = get_init_validate_status()
+        if init_status:
+            return { 'status': 'finished' }
+        return {'status': 'not_started' }
+
+    @only_edition_self_hosted
+    def post(self):
+        # is tenant created
+        tenant_count = TenantService.get_tenant_count()
+        if tenant_count > 0:
+            raise AlreadySetupError()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('password', type=str_len(30),
+                            required=True, location='json')
+        input_password = parser.parse_args()['password']
+
+        if input_password != os.environ.get('INIT_PASSWORD'):
+            session['is_init_validated'] = False
+            raise InitValidateFailedError()
+            
+        session['is_init_validated'] = True
+        return {'result': 'success'}, 201
+
+def get_init_validate_status():
+    if current_app.config['EDITION'] == 'SELF_HOSTED':
+        if os.environ.get('INIT_PASSWORD'):
+            return session.get('is_init_validated') or DifySetup.query.first()
+    
+    return True
+
+api.add_resource(InitValidateAPI, '/init')

api/controllers/console/ping.py

@@ -0,0 +1,17 @@
+from flask_restful import Resource
+
+from controllers.console import api
+
+
+class PingApi(Resource):
+
+    def get(self):
+        """
+        For connection health check
+        """
+        return {
+            "result": "pong"
+        }
+
+
+api.add_resource(PingApi, '/ping')

api/controllers/console/setup.py

@@ -1,16 +1,17 @@
-# -*- coding:utf-8 -*-
 from functools import wraps
 
-from extensions.ext_database import db
 from flask import current_app, request
 from flask_restful import Resource, reqparse
+
+from extensions.ext_database import db
 from libs.helper import email, str_len
 from libs.password import valid_password
 from models.model import DifySetup
 from services.account_service import AccountService, RegisterService, TenantService
 
 from . import api
-from .error import AlreadySetupError, NotSetupError
+from .error import AlreadySetupError, NotInitValidateError, NotSetupError
+from .init_validate import get_init_validate_status
 from .wraps import only_edition_self_hosted
 
 
@@ -24,7 +25,7 @@ class SetupApi(Resource):
                     'step': 'finished',
                     'setup_at': setup_status.setup_at.isoformat()
                 }
-            return {'step': 'not_start'}
+            return {'step': 'not_started'}
         return {'step': 'finished'}
 
     @only_edition_self_hosted
@@ -37,6 +38,9 @@ class SetupApi(Resource):
         tenant_count = TenantService.get_tenant_count()
         if tenant_count > 0:
             raise AlreadySetupError()
+    
+        if not get_init_validate_status():
+            raise NotInitValidateError()
 
         parser = reqparse.RequestParser()
         parser.add_argument('email', type=email,
@@ -71,7 +75,10 @@ def setup_required(view):
     @wraps(view)
     def decorated(*args, **kwargs):
         # check setup
-        if not get_setup_status():
+        if not get_init_validate_status():
+            raise NotInitValidateError()
+        
+        elif not get_setup_status():
             raise NotSetupError()
 
         return view(*args, **kwargs)

api/controllers/console/universal_chat/audio.py

@@ -1,64 +0,0 @@
-# -*- coding:utf-8 -*-
-import logging
-
-import services
-from controllers.console import api
-from controllers.console.app.error import (AppUnavailableError, AudioTooLargeError, CompletionRequestError,
-                                           NoAudioUploadedError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderNotSupportSpeechToTextError,
-                                           ProviderQuotaExceededError, UnsupportedAudioTypeError)
-from controllers.console.universal_chat.wraps import UniversalChatResource
-from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
-from core.model_runtime.errors.invoke import InvokeError
-from flask import request
-from models.model import AppModelConfig
-from services.audio_service import AudioService
-from services.errors.audio import (AudioTooLargeServiceError, NoAudioUploadedServiceError,
-                                   ProviderNotSupportSpeechToTextServiceError, UnsupportedAudioTypeServiceError)
-from werkzeug.exceptions import InternalServerError
-
-
-class UniversalChatAudioApi(UniversalChatResource):
-    def post(self, universal_app):
-        app_model = universal_app
-        app_model_config: AppModelConfig = app_model.app_model_config
-
-        if not app_model_config.speech_to_text_dict['enabled']:
-            raise AppUnavailableError()
-
-        file = request.files['file']
-
-        try:
-            response = AudioService.transcript(
-                tenant_id=app_model.tenant_id,
-                file=file,
-            )
-
-            return response
-        except services.errors.app_model_config.AppModelConfigBrokenError:
-            logging.exception("App model config broken.")
-            raise AppUnavailableError()
-        except NoAudioUploadedServiceError:
-            raise NoAudioUploadedError()
-        except AudioTooLargeServiceError as e:
-            raise AudioTooLargeError(str(e))
-        except UnsupportedAudioTypeServiceError:
-            raise UnsupportedAudioTypeError()
-        except ProviderNotSupportSpeechToTextServiceError:
-            raise ProviderNotSupportSpeechToTextError()
-        except ProviderTokenNotInitError:
-            raise ProviderNotInitializeError()
-        except QuotaExceededError:
-            raise ProviderQuotaExceededError()
-        except ModelCurrentlyNotSupportError:
-            raise ProviderModelCurrentlyNotSupportError()
-        except InvokeError as e:
-            raise CompletionRequestError(e.description)
-        except ValueError as e:
-            raise e
-        except Exception as e:
-            logging.exception("internal server error.")
-            raise InternalServerError()
-        
-
-api.add_resource(UniversalChatAudioApi, '/universal-chat/audio-to-text')

api/controllers/console/universal_chat/chat.py

@@ -1,141 +0,0 @@
-import json
-import logging
-from typing import Generator, Union
-
-import services
-from controllers.console import api
-from controllers.console.app.error import (AppUnavailableError, CompletionRequestError, ConversationCompletedError,
-                                           ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
-from controllers.console.universal_chat.wraps import UniversalChatResource
-from core.application_queue_manager import ApplicationQueueManager
-from core.entities.application_entities import InvokeFrom
-from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
-from core.model_runtime.errors.invoke import InvokeError
-from flask import Response, stream_with_context
-from flask_login import current_user
-from flask_restful import reqparse
-from libs.helper import uuid_value
-from services.completion_service import CompletionService
-from werkzeug.exceptions import InternalServerError, NotFound
-
-
-class UniversalChatApi(UniversalChatResource):
-    def post(self, universal_app):
-        app_model = universal_app
-
-        parser = reqparse.RequestParser()
-        parser.add_argument('query', type=str, required=True, location='json')
-        parser.add_argument('files', type=list, required=False, location='json')
-        parser.add_argument('conversation_id', type=uuid_value, location='json')
-        parser.add_argument('provider', type=str, required=True, location='json')
-        parser.add_argument('model', type=str, required=True, location='json')
-        parser.add_argument('tools', type=list, required=True, location='json')
-        parser.add_argument('retriever_from', type=str, required=False, default='universal_app', location='json')
-        args = parser.parse_args()
-
-        app_model_config = app_model.app_model_config
-
-        # update app model config
-        args['model_config'] = app_model_config.to_dict()
-        args['model_config']['model']['name'] = args['model']
-        args['model_config']['model']['provider'] = args['provider']
-        args['model_config']['agent_mode']['tools'] = args['tools']
-
-        if not args['model_config']['agent_mode']['tools']:
-            args['model_config']['agent_mode']['tools'] = [
-                {
-                    "current_datetime": {
-                        "enabled": True
-                    }
-                }
-            ]
-        else:
-            args['model_config']['agent_mode']['tools'].append({
-                    "current_datetime": {
-                        "enabled": True
-                    }
-                })
-
-        args['inputs'] = {}
-
-        del args['model']
-        del args['tools']
-
-        args['auto_generate_name'] = False
-
-        try:
-            response = CompletionService.completion(
-                app_model=app_model,
-                user=current_user,
-                args=args,
-                invoke_from=InvokeFrom.EXPLORE,
-                streaming=True,
-                is_model_config_override=True,
-            )
-
-            return compact_response(response)
-        except services.errors.conversation.ConversationNotExistsError:
-            raise NotFound("Conversation Not Exists.")
-        except services.errors.conversation.ConversationCompletedError:
-            raise ConversationCompletedError()
-        except services.errors.app_model_config.AppModelConfigBrokenError:
-            logging.exception("App model config broken.")
-            raise AppUnavailableError()
-        except ProviderTokenNotInitError:
-            raise ProviderNotInitializeError()
-        except QuotaExceededError:
-            raise ProviderQuotaExceededError()
-        except ModelCurrentlyNotSupportError:
-            raise ProviderModelCurrentlyNotSupportError()
-        except InvokeError as e:
-            raise CompletionRequestError(e.description)
-        except ValueError as e:
-            raise e
-        except Exception as e:
-            logging.exception("internal server error.")
-            raise InternalServerError()
-
-
-class UniversalChatStopApi(UniversalChatResource):
-    def post(self, universal_app, task_id):
-        ApplicationQueueManager.set_stop_flag(task_id, InvokeFrom.EXPLORE, current_user.id)
-
-        return {'result': 'success'}, 200
-
-
-def compact_response(response: Union[dict, Generator]) -> Response:
-    if isinstance(response, dict):
-        return Response(response=json.dumps(response), status=200, mimetype='application/json')
-    else:
-        def generate() -> Generator:
-            try:
-                for chunk in response:
-                    yield chunk
-            except services.errors.conversation.ConversationNotExistsError:
-                yield "data: " + json.dumps(api.handle_error(NotFound("Conversation Not Exists.")).get_json()) + "\n\n"
-            except services.errors.conversation.ConversationCompletedError:
-                yield "data: " + json.dumps(api.handle_error(ConversationCompletedError()).get_json()) + "\n\n"
-            except services.errors.app_model_config.AppModelConfigBrokenError:
-                logging.exception("App model config broken.")
-                yield "data: " + json.dumps(api.handle_error(AppUnavailableError()).get_json()) + "\n\n"
-            except ProviderTokenNotInitError:
-                yield "data: " + json.dumps(api.handle_error(ProviderNotInitializeError()).get_json()) + "\n\n"
-            except QuotaExceededError:
-                yield "data: " + json.dumps(api.handle_error(ProviderQuotaExceededError()).get_json()) + "\n\n"
-            except ModelCurrentlyNotSupportError:
-                yield "data: " + json.dumps(api.handle_error(ProviderModelCurrentlyNotSupportError()).get_json()) + "\n\n"
-            except InvokeError as e:
-                yield "data: " + json.dumps(api.handle_error(CompletionRequestError(e.description)).get_json()) + "\n\n"
-            except ValueError as e:
-                yield "data: " + json.dumps(api.handle_error(e).get_json()) + "\n\n"
-            except Exception:
-                logging.exception("internal server error.")
-                yield "data: " + json.dumps(api.handle_error(InternalServerError()).get_json()) + "\n\n"
-
-        return Response(stream_with_context(generate()), status=200,
-                        mimetype='text/event-stream')
-
-
-api.add_resource(UniversalChatApi, '/universal-chat/messages')
-api.add_resource(UniversalChatStopApi, '/universal-chat/messages/<string:task_id>/stop')

api/controllers/console/universal_chat/conversation.py

@@ -1,110 +0,0 @@
-# -*- coding:utf-8 -*-
-from controllers.console import api
-from controllers.console.universal_chat.wraps import UniversalChatResource
-from fields.conversation_fields import (conversation_with_model_config_fields,
-                                        conversation_with_model_config_infinite_scroll_pagination_fields)
-from flask_login import current_user
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
-from libs.helper import TimestampField, uuid_value
-from services.conversation_service import ConversationService
-from services.errors.conversation import ConversationNotExistsError, LastConversationNotExistsError
-from services.web_conversation_service import WebConversationService
-from werkzeug.exceptions import NotFound
-
-
-class UniversalChatConversationListApi(UniversalChatResource):
-
-    @marshal_with(conversation_with_model_config_infinite_scroll_pagination_fields)
-    def get(self, universal_app):
-        app_model = universal_app
-
-        parser = reqparse.RequestParser()
-        parser.add_argument('last_id', type=uuid_value, location='args')
-        parser.add_argument('limit', type=int_range(1, 100), required=False, default=20, location='args')
-        parser.add_argument('pinned', type=str, choices=['true', 'false', None], location='args')
-        args = parser.parse_args()
-
-        pinned = None
-        if 'pinned' in args and args['pinned'] is not None:
-            pinned = True if args['pinned'] == 'true' else False
-
-        try:
-            return WebConversationService.pagination_by_last_id(
-                app_model=app_model,
-                user=current_user,
-                last_id=args['last_id'],
-                limit=args['limit'],
-                pinned=pinned
-            )
-        except LastConversationNotExistsError:
-            raise NotFound("Last Conversation Not Exists.")
-
-
-class UniversalChatConversationApi(UniversalChatResource):
-    def delete(self, universal_app, c_id):
-        app_model = universal_app
-        conversation_id = str(c_id)
-
-        try:
-            ConversationService.delete(app_model, conversation_id, current_user)
-        except ConversationNotExistsError:
-            raise NotFound("Conversation Not Exists.")
-
-        WebConversationService.unpin(app_model, conversation_id, current_user)
-
-        return {"result": "success"}, 204
-
-
-class UniversalChatConversationRenameApi(UniversalChatResource):
-
-    @marshal_with(conversation_with_model_config_fields)
-    def post(self, universal_app, c_id):
-        app_model = universal_app
-        conversation_id = str(c_id)
-
-        parser = reqparse.RequestParser()
-        parser.add_argument('name', type=str, required=False, location='json')
-        parser.add_argument('auto_generate', type=bool, required=False, default=False, location='json')
-        args = parser.parse_args()
-
-        try:
-            return ConversationService.rename(
-                app_model,
-                conversation_id,
-                current_user,
-                args['name'],
-                args['auto_generate']
-            )
-        except ConversationNotExistsError:
-            raise NotFound("Conversation Not Exists.")
-
-
-class UniversalChatConversationPinApi(UniversalChatResource):
-
-    def patch(self, universal_app, c_id):
-        app_model = universal_app
-        conversation_id = str(c_id)
-
-        try:
-            WebConversationService.pin(app_model, conversation_id, current_user)
-        except ConversationNotExistsError:
-            raise NotFound("Conversation Not Exists.")
-
-        return {"result": "success"}
-
-
-class UniversalChatConversationUnPinApi(UniversalChatResource):
-    def patch(self, universal_app, c_id):
-        app_model = universal_app
-        conversation_id = str(c_id)
-        WebConversationService.unpin(app_model, conversation_id, current_user)
-
-        return {"result": "success"}
-
-
-api.add_resource(UniversalChatConversationRenameApi, '/universal-chat/conversations/<uuid:c_id>/name')
-api.add_resource(UniversalChatConversationListApi, '/universal-chat/conversations')
-api.add_resource(UniversalChatConversationApi, '/universal-chat/conversations/<uuid:c_id>')
-api.add_resource(UniversalChatConversationPinApi, '/universal-chat/conversations/<uuid:c_id>/pin')
-api.add_resource(UniversalChatConversationUnPinApi, '/universal-chat/conversations/<uuid:c_id>/unpin')

api/controllers/console/universal_chat/message.py

@@ -1,145 +0,0 @@
-# -*- coding:utf-8 -*-
-import logging
-
-import services
-from controllers.console import api
-from controllers.console.app.error import (CompletionRequestError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderQuotaExceededError)
-from controllers.console.explore.error import AppSuggestedQuestionsAfterAnswerDisabledError
-from controllers.console.universal_chat.wraps import UniversalChatResource
-from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
-from core.model_runtime.errors.invoke import InvokeError
-from flask_login import current_user
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
-from libs.helper import TimestampField, uuid_value
-from services.errors.conversation import ConversationNotExistsError
-from services.errors.message import MessageNotExistsError, SuggestedQuestionsAfterAnswerDisabledError
-from services.message_service import MessageService
-from werkzeug.exceptions import InternalServerError, NotFound
-
-
-class UniversalChatMessageListApi(UniversalChatResource):
-    feedback_fields = {
-        'rating': fields.String
-    }
-
-    agent_thought_fields = {
-        'id': fields.String,
-        'chain_id': fields.String,
-        'message_id': fields.String,
-        'position': fields.Integer,
-        'thought': fields.String,
-        'tool': fields.String,
-        'tool_input': fields.String,
-        'created_at': TimestampField
-    }
-
-    retriever_resource_fields = {
-        'id': fields.String,
-        'message_id': fields.String,
-        'position': fields.Integer,
-        'dataset_id': fields.String,
-        'dataset_name': fields.String,
-        'document_id': fields.String,
-        'document_name': fields.String,
-        'data_source_type': fields.String,
-        'segment_id': fields.String,
-        'score': fields.Float,
-        'hit_count': fields.Integer,
-        'word_count': fields.Integer,
-        'segment_position': fields.Integer,
-        'index_node_hash': fields.String,
-        'content': fields.String,
-        'created_at': TimestampField
-    }
-
-    message_fields = {
-        'id': fields.String,
-        'conversation_id': fields.String,
-        'inputs': fields.Raw,
-        'query': fields.String,
-        'answer': fields.String,
-        'feedback': fields.Nested(feedback_fields, attribute='user_feedback', allow_null=True),
-        'retriever_resources': fields.List(fields.Nested(retriever_resource_fields)),
-        'created_at': TimestampField,
-        'agent_thoughts': fields.List(fields.Nested(agent_thought_fields))
-    }
-
-    message_infinite_scroll_pagination_fields = {
-        'limit': fields.Integer,
-        'has_more': fields.Boolean,
-        'data': fields.List(fields.Nested(message_fields))
-    }
-
-    @marshal_with(message_infinite_scroll_pagination_fields)
-    def get(self, universal_app):
-        app_model = universal_app
-
-        parser = reqparse.RequestParser()
-        parser.add_argument('conversation_id', required=True, type=uuid_value, location='args')
-        parser.add_argument('first_id', type=uuid_value, location='args')
-        parser.add_argument('limit', type=int_range(1, 100), required=False, default=20, location='args')
-        args = parser.parse_args()
-
-        try:
-            return MessageService.pagination_by_first_id(app_model, current_user,
-                                                     args['conversation_id'], args['first_id'], args['limit'])
-        except services.errors.conversation.ConversationNotExistsError:
-            raise NotFound("Conversation Not Exists.")
-        except services.errors.message.FirstMessageNotExistsError:
-            raise NotFound("First Message Not Exists.")
-
-
-class UniversalChatMessageFeedbackApi(UniversalChatResource):
-    def post(self, universal_app, message_id):
-        app_model = universal_app
-        message_id = str(message_id)
-
-        parser = reqparse.RequestParser()
-        parser.add_argument('rating', type=str, choices=['like', 'dislike', None], location='json')
-        args = parser.parse_args()
-
-        try:
-            MessageService.create_feedback(app_model, message_id, current_user, args['rating'])
-        except services.errors.message.MessageNotExistsError:
-            raise NotFound("Message Not Exists.")
-
-        return {'result': 'success'}
-
-
-class UniversalChatMessageSuggestedQuestionApi(UniversalChatResource):
-    def get(self, universal_app, message_id):
-        app_model = universal_app
-        message_id = str(message_id)
-
-        try:
-            questions = MessageService.get_suggested_questions_after_answer(
-                app_model=app_model,
-                user=current_user,
-                message_id=message_id
-            )
-        except MessageNotExistsError:
-            raise NotFound("Message not found")
-        except ConversationNotExistsError:
-            raise NotFound("Conversation not found")
-        except SuggestedQuestionsAfterAnswerDisabledError:
-            raise AppSuggestedQuestionsAfterAnswerDisabledError()
-        except ProviderTokenNotInitError:
-            raise ProviderNotInitializeError()
-        except QuotaExceededError:
-            raise ProviderQuotaExceededError()
-        except ModelCurrentlyNotSupportError:
-            raise ProviderModelCurrentlyNotSupportError()
-        except InvokeError as e:
-            raise CompletionRequestError(e.description)
-        except Exception:
-            logging.exception("internal server error.")
-            raise InternalServerError()
-
-        return {'data': questions}
-
-
-api.add_resource(UniversalChatMessageListApi, '/universal-chat/messages')
-api.add_resource(UniversalChatMessageFeedbackApi, '/universal-chat/messages/<uuid:message_id>/feedbacks')
-api.add_resource(UniversalChatMessageSuggestedQuestionApi, '/universal-chat/messages/<uuid:message_id>/suggested-questions')

api/controllers/console/universal_chat/parameter.py

@@ -1,38 +0,0 @@
-# -*- coding:utf-8 -*-
-import json
-
-from controllers.console import api
-from controllers.console.universal_chat.wraps import UniversalChatResource
-from flask_restful import fields, marshal_with
-from models.model import App
-
-
-class UniversalChatParameterApi(UniversalChatResource):
-    """Resource for app variables."""
-    parameters_fields = {
-        'opening_statement': fields.String,
-        'suggested_questions': fields.Raw,
-        'suggested_questions_after_answer': fields.Raw,
-        'speech_to_text': fields.Raw,
-        'retriever_resource': fields.Raw,
-        'annotation_reply': fields.Raw
-    }
-
-    @marshal_with(parameters_fields)
-    def get(self, universal_app: App):
-        """Retrieve app parameters."""
-        app_model = universal_app
-        app_model_config = app_model.app_model_config
-        app_model_config.retriever_resource = json.dumps({'enabled': True})
-
-        return {
-            'opening_statement': app_model_config.opening_statement,
-            'suggested_questions': app_model_config.suggested_questions_list,
-            'suggested_questions_after_answer': app_model_config.suggested_questions_after_answer_dict,
-            'speech_to_text': app_model_config.speech_to_text_dict,
-            'retriever_resource': app_model_config.retriever_resource_dict,
-            'annotation_reply': app_model_config.annotation_reply_dict,
-        }
-
-
-api.add_resource(UniversalChatParameterApi, '/universal-chat/parameters')

api/controllers/console/universal_chat/wraps.py

@@ -1,86 +0,0 @@
-import json
-from functools import wraps
-
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
-from extensions.ext_database import db
-from flask_login import current_user
-from flask_restful import Resource
-from libs.login import login_required
-from models.model import App, AppModelConfig
-
-
-def universal_chat_app_required(view=None):
-    def decorator(view):
-        @wraps(view)
-        def decorated(*args, **kwargs):
-            # get universal chat app
-            universal_app = db.session.query(App).filter(
-                App.tenant_id == current_user.current_tenant_id,
-                App.is_universal == True
-            ).first()
-
-            if universal_app is None:
-                # create universal app if not exists
-                universal_app = App(
-                    tenant_id=current_user.current_tenant_id,
-                    name='Universal Chat',
-                    mode='chat',
-                    is_universal=True,
-                    icon='',
-                    icon_background='',
-                    api_rpm=0,
-                    api_rph=0,
-                    enable_site=False,
-                    enable_api=False,
-                    status='normal'
-                )
-
-                db.session.add(universal_app)
-                db.session.flush()
-
-                app_model_config = AppModelConfig(
-                    provider="",
-                    model_id="",
-                    configs={},
-                    opening_statement='',
-                    suggested_questions=json.dumps([]),
-                    suggested_questions_after_answer=json.dumps({'enabled': True}),
-                    speech_to_text=json.dumps({'enabled': True}),
-                    retriever_resource=json.dumps({'enabled': True}),
-                    more_like_this=None,
-                    sensitive_word_avoidance=None,
-                    model=json.dumps({
-                        "provider": "openai",
-                        "name": "gpt-3.5-turbo-16k",
-                        "completion_params": {
-                            "max_tokens": 800,
-                            "temperature": 0.8,
-                            "top_p": 1,
-                            "presence_penalty": 0,
-                            "frequency_penalty": 0
-                        }
-                    }),
-                    user_input_form=json.dumps([]),
-                    pre_prompt='',
-                    agent_mode=json.dumps({"enabled": True, "strategy": "function_call", "tools": []}),
-                )
-
-                app_model_config.app_id = universal_app.id
-                db.session.add(app_model_config)
-                db.session.flush()
-
-                universal_app.app_model_config_id = app_model_config.id
-                db.session.commit()
-
-            return view(universal_app, *args, **kwargs)
-        return decorated
-
-    if view:
-        return decorator(view)
-    return decorator
-
-
-class UniversalChatResource(Resource):
-    # must be reversed if there are multiple decorators
-    method_decorators = [universal_chat_app_required, account_initialization_required, login_required, setup_required]

api/controllers/console/version.py

@@ -1,4 +1,3 @@
-# -*- coding:utf-8 -*-
 
 import json
 import logging
@@ -6,7 +5,6 @@ import logging
 import requests
 from flask import current_app
 from flask_restful import Resource, reqparse
-from werkzeug.exceptions import InternalServerError
 
 from . import api
 

api/controllers/console/workspace/account.py

@@ -1,36 +1,28 @@
-# -*- coding:utf-8 -*-
 from datetime import datetime
 
 import pytz
-from controllers.console import api
-from controllers.console.setup import setup_required
-from controllers.console.workspace.error import (AccountAlreadyInitedError, CurrentPasswordIncorrectError,
-                                                 InvalidInvitationCodeError, RepeatPasswordNotMatchError)
-from controllers.console.wraps import account_initialization_required
-from extensions.ext_database import db
 from flask import current_app, request
 from flask_login import current_user
 from flask_restful import Resource, fields, marshal_with, reqparse
-from libs.helper import TimestampField, supported_language, timezone
+
+from constants.languages import supported_language
+from controllers.console import api
+from controllers.console.setup import setup_required
+from controllers.console.workspace.error import (
+    AccountAlreadyInitedError,
+    CurrentPasswordIncorrectError,
+    InvalidInvitationCodeError,
+    RepeatPasswordNotMatchError,
+)
+from controllers.console.wraps import account_initialization_required
+from extensions.ext_database import db
+from fields.member_fields import account_fields
+from libs.helper import TimestampField, timezone
 from libs.login import login_required
 from models.account import AccountIntegrate, InvitationCode
 from services.account_service import AccountService
 from services.errors.account import CurrentPasswordIncorrectError as ServiceCurrentPasswordIncorrectError
 
-account_fields = {
-    'id': fields.String,
-    'name': fields.String,
-    'avatar': fields.String,
-    'email': fields.String,
-    'is_password_set': fields.Boolean,
-    'interface_language': fields.String,
-    'interface_theme': fields.String,
-    'timezone': fields.String,
-    'last_login_at': TimestampField,
-    'last_login_ip': fields.String,
-    'created_at': TimestampField
-}
-
 
 class AccountInitApi(Resource):
 

api/controllers/console/workspace/members.py

@@ -1,31 +1,17 @@
-# -*- coding:utf-8 -*-
+from flask import current_app
+from flask_login import current_user
+from flask_restful import Resource, abort, marshal_with, reqparse
+
 import services
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_database import db
-from flask import current_app
-from flask_login import current_user
-from flask_restful import Resource, abort, fields, marshal, marshal_with, reqparse
-from libs.helper import TimestampField
+from fields.member_fields import account_with_role_list_fields
 from libs.login import login_required
-from models.account import Account, TenantAccountJoin
+from models.account import Account
 from services.account_service import RegisterService, TenantService
-
-account_fields = {
-    'id': fields.String,
-    'name': fields.String,
-    'avatar': fields.String,
-    'email': fields.String,
-    'last_login_at': TimestampField,
-    'created_at': TimestampField,
-    'role': fields.String,
-    'status': fields.String,
-}
-
-account_list_fields = {
-    'accounts': fields.List(fields.Nested(account_fields))
-}
+from services.errors.account import AccountAlreadyInTenantError
 
 
 class MemberListApi(Resource):
@@ -34,7 +20,7 @@ class MemberListApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @marshal_with(account_list_fields)
+    @marshal_with(account_with_role_list_fields)
     def get(self):
         members = TenantService.get_tenant_members(current_user.current_tenant)
         return {'result': 'success', 'accounts': members}, 200
@@ -51,10 +37,12 @@ class MemberInviteEmailApi(Resource):
         parser = reqparse.RequestParser()
         parser.add_argument('emails', type=str, required=True, location='json', action='append')
         parser.add_argument('role', type=str, required=True, default='admin', location='json')
+        parser.add_argument('language', type=str, required=False, location='json')
         args = parser.parse_args()
 
         invitee_emails = args['emails']
         invitee_role = args['role']
+        interface_language = args['language']
         if invitee_role not in ['admin', 'normal']:
             return {'code': 'invalid-role', 'message': 'Invalid role'}, 400
 
@@ -63,19 +51,19 @@ class MemberInviteEmailApi(Resource):
         console_web_url = current_app.config.get("CONSOLE_WEB_URL")
         for invitee_email in invitee_emails:
             try:
-                token = RegisterService.invite_new_member(inviter.current_tenant, invitee_email, role=invitee_role,
-                                                        inviter=inviter)
-                account = db.session.query(Account, TenantAccountJoin.role).join(
-                    TenantAccountJoin, Account.id == TenantAccountJoin.account_id
-                ).filter(Account.email == invitee_email).first()
-                account, role = account
+                token = RegisterService.invite_new_member(inviter.current_tenant, invitee_email, interface_language, role=invitee_role, inviter=inviter)
                 invitation_results.append({
                     'status': 'success',
                     'email': invitee_email,
                     'url': f'{console_web_url}/activate?email={invitee_email}&token={token}'
                 })
-                account = marshal(account, account_fields)
-                account['role'] = role
+            except AccountAlreadyInTenantError:
+                invitation_results.append({
+                    'status': 'success',
+                    'email': invitee_email,
+                    'url': f'{console_web_url}/signin'
+                })
+                break
             except Exception as e:
                 invitation_results.append({
                     'status': 'failed',

api/controllers/console/workspace/model_providers.py

@@ -1,18 +1,19 @@
 import io
 
+from flask import send_file
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import Forbidden
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.utils.encoders import jsonable_encoder
-from flask import send_file
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 from services.billing_service import BillingService
 from services.model_provider_service import ModelProviderService
-from werkzeug.exceptions import Forbidden
 
 
 class ModelProviderListApi(Resource):
@@ -98,7 +99,7 @@ class ModelProviderApi(Resource):
     @login_required
     @account_initialization_required
     def post(self, provider: str):
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -122,7 +123,7 @@ class ModelProviderApi(Resource):
     @login_required
     @account_initialization_required
     def delete(self, provider: str):
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         model_provider_service = ModelProviderService()
@@ -159,7 +160,7 @@ class PreferredProviderTypeUpdateApi(Resource):
     @login_required
     @account_initialization_required
     def post(self, provider: str):
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         tenant_id = current_user.current_tenant_id
@@ -186,10 +187,11 @@ class ModelProviderPaymentCheckoutUrlApi(Resource):
     def get(self, provider: str):
         if provider != 'anthropic':
             raise ValueError(f'provider name {provider} is invalid')
-
+        BillingService.is_tenant_owner_or_admin(current_user)
         data = BillingService.get_model_provider_payment_link(provider_name=provider,
                                                               tenant_id=current_user.current_tenant_id,
-                                                              account_id=current_user.id)
+                                                              account_id=current_user.id,
+                                                              prefilled_email=current_user.email)
         return data
 
 

api/controllers/console/workspace/models.py

@@ -1,16 +1,17 @@
 import logging
 
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import Forbidden
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.utils.encoders import jsonable_encoder
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 from services.model_provider_service import ModelProviderService
-from werkzeug.exceptions import Forbidden
 
 
 class DefaultModelApi(Resource):

api/controllers/console/workspace/tool_providers.py

@@ -1,136 +1,365 @@
-import json
+import io
+
+from flask import current_app, send_file
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
-from core.tool.provider.errors import ToolValidateFailedError
-from core.tool.provider.tool_provider_service import ToolProviderService
-from extensions.ext_database import db
-from flask_login import current_user
-from flask_restful import Resource, abort, reqparse
+from core.model_runtime.utils.encoders import jsonable_encoder
 from libs.login import login_required
-from models.tool import ToolProvider, ToolProviderName
-from werkzeug.exceptions import Forbidden
+from services.tools_manage_service import ToolManageService
 
 
 class ToolProviderListApi(Resource):
-
     @setup_required
     @login_required
     @account_initialization_required
     def get(self):
+        user_id = current_user.id
         tenant_id = current_user.current_tenant_id
 
-        tool_credential_dict = {}
-        for tool_name in ToolProviderName:
-            tool_credential_dict[tool_name.value] = {
-                'tool_name': tool_name.value,
-                'is_enabled': False,
-                'credentials': None
-            }
+        return ToolManageService.list_tool_providers(user_id, tenant_id)
 
-        tool_providers = db.session.query(ToolProvider).filter(ToolProvider.tenant_id == tenant_id).all()
+class ToolBuiltinProviderListToolsApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, provider):
+        user_id = current_user.id
+        tenant_id = current_user.current_tenant_id
 
-        for p in tool_providers:
-            if p.is_enabled:
-                tool_credential_dict[p.tool_name] = {
-                    'tool_name': p.tool_name,
-                    'is_enabled': p.is_enabled,
-                    'credentials': ToolProviderService(tenant_id, p.tool_name).get_credentials(obfuscated=True)
-                }
-
-        return list(tool_credential_dict.values())
-
-
-class ToolProviderCredentialsApi(Resource):
+        return jsonable_encoder(ToolManageService.list_builtin_tool_provider_tools(
+            user_id,
+            tenant_id,
+            provider,
+        ))
 
+class ToolBuiltinProviderDeleteApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
     def post(self, provider):
-        if provider not in [p.value for p in ToolProviderName]:
-            abort(404)
-
-        # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
-            raise Forbidden(f'User {current_user.id} is not authorized to update provider token, '
-                            f'current_role is {current_user.current_tenant.current_role}')
-
-        parser = reqparse.RequestParser()
-        parser.add_argument('credentials', type=dict, required=True, nullable=False, location='json')
-        args = parser.parse_args()
-
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
+        
+        user_id = current_user.id
         tenant_id = current_user.current_tenant_id
 
-        tool_provider_service = ToolProviderService(tenant_id, provider)
-
-        try:
-            tool_provider_service.credentials_validate(args['credentials'])
-        except ToolValidateFailedError as ex:
-            raise ValueError(str(ex))
-
-        encrypted_credentials = json.dumps(tool_provider_service.encrypt_credentials(args['credentials']))
-
-        tenant = current_user.current_tenant
-
-        tool_provider_model = db.session.query(ToolProvider).filter(
-                ToolProvider.tenant_id == tenant.id,
-                ToolProvider.tool_name == provider,
-            ).first()
-
-        # Only allow updating token for CUSTOM provider type
-        if tool_provider_model:
-            tool_provider_model.encrypted_credentials = encrypted_credentials
-            tool_provider_model.is_enabled = True
-        else:
-            tool_provider_model = ToolProvider(
-                tenant_id=tenant.id,
-                tool_name=provider,
-                encrypted_credentials=encrypted_credentials,
-                is_enabled=True
-            )
-            db.session.add(tool_provider_model)
-
-        db.session.commit()
-
-        return {'result': 'success'}, 201
-
-
-class ToolProviderCredentialsValidateApi(Resource):
-
+        return ToolManageService.delete_builtin_tool_provider(
+            user_id,
+            tenant_id,
+            provider,
+        )
+    
+class ToolBuiltinProviderUpdateApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
     def post(self, provider):
-        if provider not in [p.value for p in ToolProviderName]:
-            abort(404)
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
+        
+        user_id = current_user.id
+        tenant_id = current_user.current_tenant_id
 
         parser = reqparse.RequestParser()
         parser.add_argument('credentials', type=dict, required=True, nullable=False, location='json')
+
         args = parser.parse_args()
 
-        result = True
-        error = None
-
+        return ToolManageService.update_builtin_tool_provider(
+            user_id,
+            tenant_id,
+            provider,
+            args['credentials'],
+        )
+    
+class ToolBuiltinProviderGetCredentialsApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, provider):
+        user_id = current_user.id
         tenant_id = current_user.current_tenant_id
 
-        tool_provider_service = ToolProviderService(tenant_id, provider)
+        return ToolManageService.get_builtin_tool_provider_credentials(
+            user_id,
+            tenant_id,
+            provider,
+        )
 
-        try:
-            tool_provider_service.credentials_validate(args['credentials'])
-        except ToolValidateFailedError as ex:
-            result = False
-            error = str(ex)
+class ToolBuiltinProviderIconApi(Resource):
+    @setup_required
+    def get(self, provider):
+        icon_bytes, mimetype = ToolManageService.get_builtin_tool_provider_icon(provider)
+        icon_cache_max_age = int(current_app.config.get('TOOL_ICON_CACHE_MAX_AGE'))
+        return send_file(io.BytesIO(icon_bytes), mimetype=mimetype, max_age=icon_cache_max_age)
 
-        response = {'result': 'success' if result else 'error'}
+class ToolModelProviderIconApi(Resource):
+    @setup_required
+    def get(self, provider):
+        icon_bytes, mimetype = ToolManageService.get_model_tool_provider_icon(provider)
+        return send_file(io.BytesIO(icon_bytes), mimetype=mimetype)
+    
+class ToolModelProviderListToolsApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        user_id = current_user.id
+        tenant_id = current_user.current_tenant_id
 
-        if not result:
-            response['error'] = error
+        parser = reqparse.RequestParser()
+        parser.add_argument('provider', type=str, required=True, nullable=False, location='args')
 
-        return response
+        args = parser.parse_args()
 
+        return jsonable_encoder(ToolManageService.list_model_tool_provider_tools(
+            user_id,
+            tenant_id,
+            args['provider'],
+        ))
+
+class ToolApiProviderAddApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
+        
+        user_id = current_user.id
+        tenant_id = current_user.current_tenant_id
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('credentials', type=dict, required=True, nullable=False, location='json')
+        parser.add_argument('schema_type', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('schema', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('provider', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('icon', type=dict, required=True, nullable=False, location='json')
+        parser.add_argument('privacy_policy', type=str, required=False, nullable=True, location='json')
+
+        args = parser.parse_args()
+
+        return ToolManageService.create_api_tool_provider(
+            user_id,
+            tenant_id,
+            args['provider'],
+            args['icon'],
+            args['credentials'],
+            args['schema_type'],
+            args['schema'],
+            args.get('privacy_policy', ''),
+        )
+
+class ToolApiProviderGetRemoteSchemaApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        parser = reqparse.RequestParser()
+
+        parser.add_argument('url', type=str, required=True, nullable=False, location='args')
+
+        args = parser.parse_args()
+
+        return ToolManageService.get_api_tool_provider_remote_schema(
+            current_user.id,
+            current_user.current_tenant_id,
+            args['url'],
+        )
+    
+class ToolApiProviderListToolsApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        user_id = current_user.id
+        tenant_id = current_user.current_tenant_id
+
+        parser = reqparse.RequestParser()
+
+        parser.add_argument('provider', type=str, required=True, nullable=False, location='args')
+
+        args = parser.parse_args()
+
+        return jsonable_encoder(ToolManageService.list_api_tool_provider_tools(
+            user_id,
+            tenant_id,
+            args['provider'],
+        ))
+
+class ToolApiProviderUpdateApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
+        
+        user_id = current_user.id
+        tenant_id = current_user.current_tenant_id
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('credentials', type=dict, required=True, nullable=False, location='json')
+        parser.add_argument('schema_type', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('schema', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('provider', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('original_provider', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('icon', type=dict, required=True, nullable=False, location='json')
+        parser.add_argument('privacy_policy', type=str, required=True, nullable=True, location='json')
+
+        args = parser.parse_args()
+
+        return ToolManageService.update_api_tool_provider(
+            user_id,
+            tenant_id,
+            args['provider'],
+            args['original_provider'],
+            args['icon'],
+            args['credentials'],
+            args['schema_type'],
+            args['schema'],
+            args['privacy_policy'],
+        )
+
+class ToolApiProviderDeleteApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
+        
+        user_id = current_user.id
+        tenant_id = current_user.current_tenant_id
+
+        parser = reqparse.RequestParser()
+
+        parser.add_argument('provider', type=str, required=True, nullable=False, location='json')
+
+        args = parser.parse_args()
+
+        return ToolManageService.delete_api_tool_provider(
+            user_id,
+            tenant_id,
+            args['provider'],
+        )
+
+class ToolApiProviderGetApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        user_id = current_user.id
+        tenant_id = current_user.current_tenant_id
+
+        parser = reqparse.RequestParser()
+
+        parser.add_argument('provider', type=str, required=True, nullable=False, location='args')
+
+        args = parser.parse_args()
+
+        return ToolManageService.get_api_tool_provider(
+            user_id,
+            tenant_id,
+            args['provider'],
+        )
+
+class ToolBuiltinProviderCredentialsSchemaApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, provider):
+        return ToolManageService.list_builtin_provider_credentials_schema(provider)
+
+class ToolApiProviderSchemaApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        parser = reqparse.RequestParser()
+
+        parser.add_argument('schema', type=str, required=True, nullable=False, location='json')
+
+        args = parser.parse_args()
+
+        return ToolManageService.parser_api_schema(
+            schema=args['schema'],
+        )
+
+class ToolApiProviderPreviousTestApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        parser = reqparse.RequestParser()
+
+        parser.add_argument('tool_name', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('provider_name', type=str, required=False, nullable=False, location='json')
+        parser.add_argument('credentials', type=dict, required=True, nullable=False, location='json')
+        parser.add_argument('parameters', type=dict, required=True, nullable=False, location='json')
+        parser.add_argument('schema_type', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('schema', type=str, required=True, nullable=False, location='json')
+
+        args = parser.parse_args()
+
+        return ToolManageService.test_api_tool_preview(
+            current_user.current_tenant_id,
+            args['provider_name'] if args['provider_name'] else '',
+            args['tool_name'],
+            args['credentials'],
+            args['parameters'],
+            args['schema_type'],
+            args['schema'],
+        )
+
+class ToolBuiltinListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        user_id = current_user.id
+        tenant_id = current_user.current_tenant_id
+
+        return jsonable_encoder([provider.to_dict() for provider in ToolManageService.list_builtin_tools(
+            user_id,
+            tenant_id,
+        )])
+    
+class ToolApiListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        user_id = current_user.id
+        tenant_id = current_user.current_tenant_id
+
+        return jsonable_encoder([provider.to_dict() for provider in ToolManageService.list_api_tools(
+            user_id,
+            tenant_id,
+        )])
 
 api.add_resource(ToolProviderListApi, '/workspaces/current/tool-providers')
-api.add_resource(ToolProviderCredentialsApi, '/workspaces/current/tool-providers/<provider>/credentials')
-api.add_resource(ToolProviderCredentialsValidateApi,
-                 '/workspaces/current/tool-providers/<provider>/credentials-validate')
+api.add_resource(ToolBuiltinProviderListToolsApi, '/workspaces/current/tool-provider/builtin/<provider>/tools')
+api.add_resource(ToolBuiltinProviderDeleteApi, '/workspaces/current/tool-provider/builtin/<provider>/delete')
+api.add_resource(ToolBuiltinProviderUpdateApi, '/workspaces/current/tool-provider/builtin/<provider>/update')
+api.add_resource(ToolBuiltinProviderGetCredentialsApi, '/workspaces/current/tool-provider/builtin/<provider>/credentials')
+api.add_resource(ToolBuiltinProviderCredentialsSchemaApi, '/workspaces/current/tool-provider/builtin/<provider>/credentials_schema')
+api.add_resource(ToolBuiltinProviderIconApi, '/workspaces/current/tool-provider/builtin/<provider>/icon')
+api.add_resource(ToolModelProviderIconApi, '/workspaces/current/tool-provider/model/<provider>/icon')
+api.add_resource(ToolModelProviderListToolsApi, '/workspaces/current/tool-provider/model/tools')
+api.add_resource(ToolApiProviderAddApi, '/workspaces/current/tool-provider/api/add')
+api.add_resource(ToolApiProviderGetRemoteSchemaApi, '/workspaces/current/tool-provider/api/remote')
+api.add_resource(ToolApiProviderListToolsApi, '/workspaces/current/tool-provider/api/tools')
+api.add_resource(ToolApiProviderUpdateApi, '/workspaces/current/tool-provider/api/update')
+api.add_resource(ToolApiProviderDeleteApi, '/workspaces/current/tool-provider/api/delete')
+api.add_resource(ToolApiProviderGetApi, '/workspaces/current/tool-provider/api/get')
+api.add_resource(ToolApiProviderSchemaApi, '/workspaces/current/tool-provider/api/schema')
+api.add_resource(ToolApiProviderPreviousTestApi, '/workspaces/current/tool-provider/api/test/pre')
+
+api.add_resource(ToolBuiltinListApi, '/workspaces/current/tools/builtin')
+api.add_resource(ToolApiListApi, '/workspaces/current/tools/api')

api/controllers/console/workspace/workspace.py

@@ -1,18 +1,22 @@
-# -*- coding:utf-8 -*-
 import logging
 
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, fields, inputs, marshal, marshal_with, reqparse
+
 import services
 from controllers.console import api
 from controllers.console.admin import admin_required
-from controllers.console.datasets.error import (FileTooLargeError, NoFileUploadedError, TooManyFilesError,
-                                                UnsupportedFileTypeError)
+from controllers.console.datasets.error import (
+    FileTooLargeError,
+    NoFileUploadedError,
+    TooManyFilesError,
+    UnsupportedFileTypeError,
+)
 from controllers.console.error import AccountNotLinkTenantError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_database import db
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, fields, inputs, marshal, marshal_with, reqparse
 from libs.helper import TimestampField
 from libs.login import login_required
 from models.account import Tenant

api/controllers/console/wraps.py

@@ -1,10 +1,12 @@
-# -*- coding:utf-8 -*-
+import json
 from functools import wraps
 
-from controllers.console.workspace.error import AccountNotInitializedError
-from flask import abort, current_app
+from flask import abort, current_app, request
 from flask_login import current_user
+
+from controllers.console.workspace.error import AccountNotInitializedError
 from services.feature_service import FeatureService
+from services.operation_service import OperationService
 
 
 def account_initialization_required(view):
@@ -49,19 +51,25 @@ def cloud_edition_billing_resource_check(resource: str,
         @wraps(view)
         def decorated(*args, **kwargs):
             features = FeatureService.get_features(current_user.current_tenant_id)
-
             if features.billing.enabled:
                 members = features.members
                 apps = features.apps
                 vector_space = features.vector_space
+                documents_upload_quota = features.documents_upload_quota
                 annotation_quota_limit = features.annotation_quota_limit
-
                 if resource == 'members' and 0 < members.limit <= members.size:
                     abort(403, error_msg)
                 elif resource == 'apps' and 0 < apps.limit <= apps.size:
                     abort(403, error_msg)
                 elif resource == 'vector_space' and 0 < vector_space.limit <= vector_space.size:
                     abort(403, error_msg)
+                elif resource == 'documents' and 0 < documents_upload_quota.limit <= documents_upload_quota.size:
+                    # The api of file upload is used in the multiple places, so we need to check the source of the request from datasets
+                    source = request.args.get('source')
+                    if source == 'datasets':
+                        abort(403, error_msg)
+                    else:
+                        return view(*args, **kwargs)
                 elif resource == 'workspace_custom' and not features.can_replace_logo:
                     abort(403, error_msg)
                 elif resource == 'annotation' and 0 < annotation_quota_limit.limit < annotation_quota_limit.size:
@@ -70,6 +78,46 @@ def cloud_edition_billing_resource_check(resource: str,
                     return view(*args, **kwargs)
 
             return view(*args, **kwargs)
+
         return decorated
+
     return interceptor
 
+
+def cloud_edition_billing_knowledge_limit_check(resource: str,
+                                                error_msg: str = "To unlock this feature and elevate your Dify experience, please upgrade to a paid plan."):
+    def interceptor(view):
+        @wraps(view)
+        def decorated(*args, **kwargs):
+            features = FeatureService.get_features(current_user.current_tenant_id)
+            if features.billing.enabled:
+                if resource == 'add_segment':
+                    if features.billing.subscription.plan == 'sandbox':
+                        abort(403, error_msg)
+                else:
+                    return view(*args, **kwargs)
+
+            return view(*args, **kwargs)
+
+        return decorated
+
+    return interceptor
+
+
+def cloud_utm_record(view):
+    @wraps(view)
+    def decorated(*args, **kwargs):
+        try:
+            features = FeatureService.get_features(current_user.current_tenant_id)
+
+            if features.billing.enabled:
+                utm_info = request.cookies.get('utm_info')
+
+                if utm_info:
+                    utm_info = json.loads(utm_info)
+                    OperationService.record_utm(current_user.current_tenant_id, utm_info)
+        except Exception as e:
+            pass
+        return view(*args, **kwargs)
+
+    return decorated

api/controllers/files/__init__.py

@@ -6,4 +6,4 @@ bp = Blueprint('files', __name__)
 api = ExternalApi(bp)
 
 
-from . import image_preview
+from . import image_preview, tool_files