feat: add node id in agent log

Signed-off-by: YoungLH <974840768@qq.com>

.github/workflows/api-tests.yml

@@ -50,9 +50,18 @@ jobs:
       - name: Run ModelRuntime
         run: poetry run -C api bash dev/pytest/pytest_model_runtime.sh
 
+      - name: Run dify config tests
+        run: poetry run -C api python dev/pytest/pytest_config_tests.py
+
       - name: Run Tool
         run: poetry run -C api bash dev/pytest/pytest_tools.sh
 
+      - name: Run mypy
+        run: |
+          pushd api
+          poetry run python -m mypy --install-types --non-interactive .
+          popd
+
       - name: Set up dotenvs
         run: |
           cp docker/.env.example docker/.env

.github/workflows/expose_service_ports.sh

@@ -9,5 +9,6 @@ yq eval '.services["pgvecto-rs"].ports += ["5431:5432"]' -i docker/docker-compos
 yq eval '.services["elasticsearch"].ports += ["9200:9200"]' -i docker/docker-compose.yaml
 yq eval '.services.couchbase-server.ports += ["8091-8096:8091-8096"]' -i docker/docker-compose.yaml
 yq eval '.services.couchbase-server.ports += ["11210:11210"]' -i docker/docker-compose.yaml
+yq eval '.services.tidb.ports += ["4000:4000"]' -i docker/docker-compose.yaml
 
-echo "Ports exposed for sandbox, weaviate, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch, couchbase"
+echo "Ports exposed for sandbox, weaviate, tidb, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch, couchbase"

.gitignore

@@ -175,6 +175,7 @@ docker/volumes/pgvector/data/*
 docker/volumes/pgvecto_rs/data/*
 docker/volumes/couchbase/*
 docker/volumes/oceanbase/*
+docker/volumes/plugin_daemon/*
 !docker/volumes/oceanbase/init.d
 
 docker/nginx/conf.d/default.conf

api/.dockerignore

@@ -1,7 +1,10 @@
 .env
 *.env.*
 
+storage/generate_files/*
 storage/privkeys/*
+storage/tools/*
+storage/upload_files/*
 
 # Logs
 logs
@@ -9,6 +12,8 @@ logs
 
 # jetbrains
 .idea
+.mypy_cache
+.ruff_cache
 
 # venv
 .venv

api/.env.example

@@ -23,6 +23,9 @@ FILES_ACCESS_TIMEOUT=300
 # Access token expiration time in minutes
 ACCESS_TOKEN_EXPIRE_MINUTES=60
 
+# Refresh token expiration time in days
+REFRESH_TOKEN_EXPIRE_DAYS=30
+
 # celery configuration
 CELERY_BROKER_URL=redis://:difyai123456@localhost:6379/1
 
@@ -60,21 +63,12 @@ DB_DATABASE=dify
 STORAGE_TYPE=opendal
 
 # Apache OpenDAL storage configuration, refer to https://github.com/apache/opendal
-STORAGE_OPENDAL_SCHEME=fs
-# OpenDAL FS
+OPENDAL_SCHEME=fs
 OPENDAL_FS_ROOT=storage
-# OpenDAL S3
-OPENDAL_S3_ROOT=/
-OPENDAL_S3_BUCKET=your-bucket-name
-OPENDAL_S3_ENDPOINT=https://s3.amazonaws.com
-OPENDAL_S3_ACCESS_KEY_ID=your-access-key
-OPENDAL_S3_SECRET_ACCESS_KEY=your-secret-key
-OPENDAL_S3_REGION=your-region
-OPENDAL_S3_SERVER_SIDE_ENCRYPTION=
 
 # S3 Storage configuration
 S3_USE_AWS_MANAGED_IAM=false
-S3_ENDPOINT=https://your-bucket-name.storage.s3.clooudflare.com
+S3_ENDPOINT=https://your-bucket-name.storage.s3.cloudflare.com
 S3_BUCKET_NAME=your-bucket-name
 S3_ACCESS_KEY=your-access-key
 S3_SECRET_KEY=your-secret-key
@@ -83,7 +77,7 @@ S3_REGION=your-region
 # Azure Blob Storage configuration
 AZURE_BLOB_ACCOUNT_NAME=your-account-name
 AZURE_BLOB_ACCOUNT_KEY=your-account-key
-AZURE_BLOB_CONTAINER_NAME=yout-container-name
+AZURE_BLOB_CONTAINER_NAME=your-container-name
 AZURE_BLOB_ACCOUNT_URL=https://<your_account_name>.blob.core.windows.net
 
 # Aliyun oss Storage configuration
@@ -97,7 +91,7 @@ ALIYUN_OSS_REGION=your-region
 ALIYUN_OSS_PATH=your-path
 
 # Google Storage configuration
-GOOGLE_STORAGE_BUCKET_NAME=yout-bucket-name
+GOOGLE_STORAGE_BUCKET_NAME=your-bucket-name
 GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64=your-google-service-account-json-base64-string
 
 # Tencent COS Storage configuration
@@ -313,8 +307,7 @@ UPLOAD_VIDEO_FILE_SIZE_LIMIT=100
 UPLOAD_AUDIO_FILE_SIZE_LIMIT=50
 
 # Model configuration
-MULTIMODAL_SEND_IMAGE_FORMAT=base64
-MULTIMODAL_SEND_VIDEO_FORMAT=base64
+MULTIMODAL_SEND_FORMAT=base64
 PROMPT_GENERATION_MAX_TOKENS=512
 CODE_GENERATION_MAX_TOKENS=1024
 
@@ -409,13 +402,13 @@ INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000
 WORKFLOW_MAX_EXECUTION_STEPS=500
 WORKFLOW_MAX_EXECUTION_TIME=1200
 WORKFLOW_CALL_MAX_DEPTH=5
+WORKFLOW_PARALLEL_DEPTH_LIMIT=3
 MAX_VARIABLE_SIZE=204800
 
 # App configuration
 APP_MAX_EXECUTION_TIME=1200
 APP_MAX_ACTIVE_REQUESTS=0
 
-
 # Celery beat configuration
 CELERY_BEAT_SCHEDULER_TIME=1
 
@@ -428,6 +421,22 @@ POSITION_PROVIDER_PINS=
 POSITION_PROVIDER_INCLUDES=
 POSITION_PROVIDER_EXCLUDES=
 
+# Plugin configuration
+PLUGIN_API_KEY=lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1
+PLUGIN_API_URL=http://127.0.0.1:5002
+PLUGIN_REMOTE_INSTALL_PORT=5003
+PLUGIN_REMOTE_INSTALL_HOST=localhost
+PLUGIN_MAX_PACKAGE_SIZE=15728640
+INNER_API_KEY=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1
+INNER_API_KEY_FOR_PLUGIN=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1
+
+# Marketplace configuration
+MARKETPLACE_ENABLED=true
+MARKETPLACE_API_URL=https://marketplace.dify.ai
+
+# Endpoint configuration
+ENDPOINT_URL_TEMPLATE=http://localhost/e/{hook_id}
+
 # Reset password token expiry minutes
 RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5
 
@@ -435,3 +444,5 @@ CREATE_TIDB_SERVICE_JOB_ENABLED=false
 
 # Maximum number of submitted thread count in a ThreadPool for parallel node execution
 MAX_SUBMIT_COUNT=100
+# Lockout duration in seconds
+LOGIN_LOCKOUT_DURATION=86400

api/.ruff.toml

@@ -67,10 +67,9 @@ ignore = [
     "SIM105", # suppressible-exception
     "SIM107", # return-in-try-except-finally
     "SIM108", # if-else-block-instead-of-if-exp
-    "SIM113", # eumerate-for-loop
+    "SIM113", # enumerate-for-loop
     "SIM117", # multiple-with-statements
     "SIM210", # if-expr-with-true-false
-    "SIM300", # yoda-conditions,
 ]
 
 [lint.per-file-ignores]
@@ -86,11 +85,11 @@ ignore = [
 ]
 "tests/*" = [
     "F811", # redefined-while-unused
-    "F401", # unused-import
 ]
 
 [lint.pyflakes]
-extend-generics = [
+allowed-unused-imports = [
     "_pytest.monkeypatch",
     "tests.integration_tests",
+    "tests.unit_tests",
 ]

api/Dockerfile

@@ -55,7 +55,7 @@ RUN apt-get update \
     && echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \
     && apt-get update \
     # For Security
-    && apt-get install -y --no-install-recommends expat=2.6.4-1 libldap-2.5-0=2.5.18+dfsg-3+b1 perl=5.40.0-8 libsqlite3-0=3.46.1-1 zlib1g=1:1.3.dfsg+really1.3.1-1+b1 \
+    && apt-get install -y --no-install-recommends expat=2.6.4-1 libldap-2.5-0=2.5.19+dfsg-1 perl=5.40.0-8 libsqlite3-0=3.46.1-1 zlib1g=1:1.3.dfsg+really1.3.1-1+b1 \
     # install a chinese font to support the use of tools like matplotlib
     && apt-get install -y fonts-noto-cjk \
     && apt-get autoremove -y \
@@ -69,6 +69,10 @@ ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 # Download nltk data
 RUN python -c "import nltk; nltk.download('punkt'); nltk.download('averaged_perceptron_tagger')"
 
+ENV TIKTOKEN_CACHE_DIR=/app/api/.tiktoken_cache
+
+RUN python -c "import tiktoken; tiktoken.encoding_for_model('gpt2')"
+
 # Copy source code
 COPY . /app/api/
 

api/app.py

@@ -1,13 +1,41 @@
-from app_factory import create_app
-from libs import threadings_utils, version_utils
+import os
+import sys
+
+
+def is_db_command():
+    if len(sys.argv) > 1 and sys.argv[0].endswith("flask") and sys.argv[1] == "db":
+        return True
+    return False
 
-# preparation before creating app
-version_utils.check_supported_python_version()
-threadings_utils.apply_gevent_threading_patch()
 
 # create app
-app = create_app()
-celery = app.extensions["celery"]
+if is_db_command():
+    from app_factory import create_migrations_app
+
+    app = create_migrations_app()
+else:
+    # It seems that JetBrains Python debugger does not work well with gevent,
+    # so we need to disable gevent in debug mode.
+    # If you are using debugpy and set GEVENT_SUPPORT=True, you can debug with gevent.
+    if (flask_debug := os.environ.get("FLASK_DEBUG", "0")) and flask_debug.lower() in {"false", "0", "no"}:
+        from gevent import monkey  # type: ignore
+
+        # gevent
+        monkey.patch_all()
+
+        from grpc.experimental import gevent as grpc_gevent  # type: ignore
+
+        # grpc gevent
+        grpc_gevent.init_gevent()
+
+        import psycogreen.gevent  # type: ignore
+
+        psycogreen.gevent.patch_psycopg()
+
+    from app_factory import create_app
+
+    app = create_app()
+    celery = app.extensions["celery"]
 
 if __name__ == "__main__":
     app.run(host="0.0.0.0", port=5001)

api/app_factory.py

@@ -1,5 +1,4 @@
 import logging
-import os
 import time
 
 from configs import dify_config
@@ -17,15 +16,6 @@ def create_flask_app_with_configs() -> DifyApp:
     dify_app = DifyApp(__name__)
     dify_app.config.from_mapping(dify_config.model_dump())
 
-    # populate configs into system environment variables
-    for key, value in dify_app.config.items():
-        if isinstance(value, str):
-            os.environ[key] = value
-        elif isinstance(value, int | float | bool):
-            os.environ[key] = str(value)
-        elif value is None:
-            os.environ[key] = ""
-
     return dify_app
 
 
@@ -98,3 +88,14 @@ def initialize_extensions(app: DifyApp):
         end_time = time.perf_counter()
         if dify_config.DEBUG:
             logging.info(f"Loaded {short_name} ({round((end_time - start_time) * 1000, 2)} ms)")
+
+
+def create_migrations_app():
+    app = create_flask_app_with_configs()
+    from extensions import ext_database, ext_migrate
+
+    # Initialize only required extensions
+    ext_database.init_app(app)
+    ext_migrate.init_app(app)
+
+    return app

api/commands.py

@@ -25,6 +25,8 @@ from models.dataset import Document as DatasetDocument
 from models.model import Account, App, AppAnnotationSetting, AppMode, Conversation, MessageAnnotation
 from models.provider import Provider, ProviderModel
 from services.account_service import RegisterService, TenantService
+from services.plugin.data_migration import PluginDataMigration
+from services.plugin.plugin_migration import PluginMigration
 
 
 @click.command("reset-password", help="Reset the account password.")
@@ -159,8 +161,7 @@ def migrate_annotation_vector_database():
         try:
             # get apps info
             apps = (
-                db.session.query(App)
-                .filter(App.status == "normal")
+                App.query.filter(App.status == "normal")
                 .order_by(App.created_at.desc())
                 .paginate(page=page, per_page=50)
             )
@@ -285,8 +286,7 @@ def migrate_knowledge_vector_database():
     while True:
         try:
             datasets = (
-                db.session.query(Dataset)
-                .filter(Dataset.indexing_technique == "high_quality")
+                Dataset.query.filter(Dataset.indexing_technique == "high_quality")
                 .order_by(Dataset.created_at.desc())
                 .paginate(page=page, per_page=50)
             )
@@ -450,7 +450,8 @@ def convert_to_agent_apps():
                 if app_id not in proceeded_app_ids:
                     proceeded_app_ids.append(app_id)
                     app = db.session.query(App).filter(App.id == app_id).first()
-                    apps.append(app)
+                    if app is not None:
+                        apps.append(app)
 
             if len(apps) == 0:
                 break
@@ -525,7 +526,7 @@ def add_qdrant_doc_id_index(field: str):
                         )
                     )
 
-    except Exception as e:
+    except Exception:
         click.echo(click.style("Failed to create Qdrant client.", fg="red"))
 
     click.echo(click.style(f"Index creation complete. Created {create_count} collection indexes.", fg="green"))
@@ -555,14 +556,20 @@ def create_tenant(email: str, language: Optional[str] = None, name: Optional[str
     if language not in languages:
         language = "en-US"
 
-    name = name.strip()
+    # Validates name encoding for non-Latin characters.
+    name = name.strip().encode("utf-8").decode("utf-8") if name else None
 
     # generate random password
     new_password = secrets.token_urlsafe(16)
 
     # register account
-    account = RegisterService.register(email=email, name=account_name, password=new_password, language=language)
-
+    account = RegisterService.register(
+        email=email,
+        name=account_name,
+        password=new_password,
+        language=language,
+        create_workspace_required=False,
+    )
     TenantService.create_owner_tenant_if_not_exist(account, name)
 
     click.echo(
@@ -582,13 +589,13 @@ def upgrade_db():
             click.echo(click.style("Starting database migration.", fg="green"))
 
             # run db migration
-            import flask_migrate
+            import flask_migrate  # type: ignore
 
             flask_migrate.upgrade()
 
             click.echo(click.style("Database migration successful!", fg="green"))
 
-        except Exception as e:
+        except Exception:
             logging.exception("Failed to execute database migration")
         finally:
             lock.release()
@@ -620,6 +627,10 @@ where sites.id is null limit 1000"""
 
                 try:
                     app = db.session.query(App).filter(App.id == app_id).first()
+                    if not app:
+                        print(f"App {app_id} not found")
+                        continue
+
                     tenant = app.tenant
                     if tenant:
                         accounts = tenant.get_accounts()
@@ -630,7 +641,7 @@ where sites.id is null limit 1000"""
                         account = accounts[0]
                         print("Fixing missing site for app {}".format(app.id))
                         app_was_created.send(app, account=account)
-                except Exception as e:
+                except Exception:
                     failed_app_ids.append(app_id)
                     click.echo(click.style("Failed to fix missing site for app {}".format(app_id), fg="red"))
                     logging.exception(f"Failed to fix app related site missing issue, app_id: {app_id}")
@@ -640,3 +651,68 @@ where sites.id is null limit 1000"""
                 break
 
     click.echo(click.style("Fix for missing app-related sites completed successfully!", fg="green"))
+
+
+@click.command("migrate-data-for-plugin", help="Migrate data for plugin.")
+def migrate_data_for_plugin():
+    """
+    Migrate data for plugin.
+    """
+    click.echo(click.style("Starting migrate data for plugin.", fg="white"))
+
+    PluginDataMigration.migrate()
+
+    click.echo(click.style("Migrate data for plugin completed.", fg="green"))
+
+
+@click.command("extract-plugins", help="Extract plugins.")
+@click.option("--output_file", prompt=True, help="The file to store the extracted plugins.", default="plugins.jsonl")
+@click.option("--workers", prompt=True, help="The number of workers to extract plugins.", default=10)
+def extract_plugins(output_file: str, workers: int):
+    """
+    Extract plugins.
+    """
+    click.echo(click.style("Starting extract plugins.", fg="white"))
+
+    PluginMigration.extract_plugins(output_file, workers)
+
+    click.echo(click.style("Extract plugins completed.", fg="green"))
+
+
+@click.command("extract-unique-identifiers", help="Extract unique identifiers.")
+@click.option(
+    "--output_file",
+    prompt=True,
+    help="The file to store the extracted unique identifiers.",
+    default="unique_identifiers.json",
+)
+@click.option(
+    "--input_file", prompt=True, help="The file to store the extracted unique identifiers.", default="plugins.jsonl"
+)
+def extract_unique_plugins(output_file: str, input_file: str):
+    """
+    Extract unique plugins.
+    """
+    click.echo(click.style("Starting extract unique plugins.", fg="white"))
+
+    PluginMigration.extract_unique_plugins_to_file(input_file, output_file)
+
+    click.echo(click.style("Extract unique plugins completed.", fg="green"))
+
+
+@click.command("install-plugins", help="Install plugins.")
+@click.option(
+    "--input_file", prompt=True, help="The file to store the extracted unique identifiers.", default="plugins.jsonl"
+)
+@click.option(
+    "--output_file", prompt=True, help="The file to store the installed plugins.", default="installed_plugins.jsonl"
+)
+def install_plugins(input_file: str, output_file: str):
+    """
+    Install plugins.
+    """
+    click.echo(click.style("Starting install plugins.", fg="white"))
+
+    PluginMigration.install_plugins(input_file, output_file)
+
+    click.echo(click.style("Install plugins completed.", fg="green"))

api/configs/feature/__init__.py

@@ -134,6 +134,60 @@ class CodeExecutionSandboxConfig(BaseSettings):
     )
 
 
+class PluginConfig(BaseSettings):
+    """
+    Plugin configs
+    """
+
+    PLUGIN_DAEMON_URL: HttpUrl = Field(
+        description="Plugin API URL",
+        default="http://plugin:5002",
+    )
+
+    PLUGIN_API_KEY: str = Field(
+        description="Plugin API key",
+        default="plugin-api-key",
+    )
+
+    INNER_API_KEY_FOR_PLUGIN: str = Field(description="Inner api key for plugin", default="inner-api-key")
+
+    PLUGIN_REMOTE_INSTALL_HOST: str = Field(
+        description="Plugin Remote Install Host",
+        default="localhost",
+    )
+
+    PLUGIN_REMOTE_INSTALL_PORT: PositiveInt = Field(
+        description="Plugin Remote Install Port",
+        default=5003,
+    )
+
+    PLUGIN_MAX_PACKAGE_SIZE: PositiveInt = Field(
+        description="Maximum allowed size for plugin packages in bytes",
+        default=15728640,
+    )
+
+    PLUGIN_MAX_BUNDLE_SIZE: PositiveInt = Field(
+        description="Maximum allowed size for plugin bundles in bytes",
+        default=15728640 * 12,
+    )
+
+
+class MarketplaceConfig(BaseSettings):
+    """
+    Configuration for marketplace
+    """
+
+    MARKETPLACE_ENABLED: bool = Field(
+        description="Enable or disable marketplace",
+        default=True,
+    )
+
+    MARKETPLACE_API_URL: HttpUrl = Field(
+        description="Marketplace API URL",
+        default="https://marketplace.dify.ai",
+    )
+
+
 class EndpointConfig(BaseSettings):
     """
     Configuration for various application endpoints and URLs
@@ -160,6 +214,10 @@ class EndpointConfig(BaseSettings):
         default="",
     )
 
+    ENDPOINT_URL_TEMPLATE: str = Field(
+        description="Template url for endpoint plugin", default="http://localhost:5002/e/{hook_id}"
+    )
+
 
 class FileAccessConfig(BaseSettings):
     """
@@ -239,7 +297,6 @@ class HttpConfig(BaseSettings):
     )
 
     @computed_field
-    @property
     def CONSOLE_CORS_ALLOW_ORIGINS(self) -> list[str]:
         return self.inner_CONSOLE_CORS_ALLOW_ORIGINS.split(",")
 
@@ -250,7 +307,6 @@ class HttpConfig(BaseSettings):
     )
 
     @computed_field
-    @property
     def WEB_API_CORS_ALLOW_ORIGINS(self) -> list[str]:
         return self.inner_WEB_API_CORS_ALLOW_ORIGINS.split(",")
 
@@ -433,6 +489,11 @@ class WorkflowConfig(BaseSettings):
         default=5,
     )
 
+    WORKFLOW_PARALLEL_DEPTH_LIMIT: PositiveInt = Field(
+        description="Maximum allowed depth for nested parallel executions",
+        default=3,
+    )
+
     MAX_VARIABLE_SIZE: PositiveInt = Field(
         description="Maximum size in bytes for a single variable in workflows. Default to 200 KB.",
         default=200 * 1024,
@@ -485,6 +546,16 @@ class AuthConfig(BaseSettings):
         default=60,
     )
 
+    REFRESH_TOKEN_EXPIRE_DAYS: PositiveFloat = Field(
+        description="Expiration time for refresh tokens in days",
+        default=30,
+    )
+
+    LOGIN_LOCKOUT_DURATION: PositiveInt = Field(
+        description="Time (in seconds) a user must wait before retrying login after exceeding the rate limit.",
+        default=86400,
+    )
+
 
 class ModerationConfig(BaseSettings):
     """
@@ -593,7 +664,7 @@ class RagEtlConfig(BaseSettings):
 
     UNSTRUCTURED_API_KEY: Optional[str] = Field(
         description="API key for Unstructured.io service",
-        default=None,
+        default="",
     )
 
     SCARF_NO_ANALYTICS: Optional[str] = Field(
@@ -659,15 +730,15 @@ class IndexingConfig(BaseSettings):
         default=4000,
     )
 
-
-class VisionFormatConfig(BaseSettings):
-    MULTIMODAL_SEND_IMAGE_FORMAT: Literal["base64", "url"] = Field(
-        description="Format for sending images in multimodal contexts ('base64' or 'url'), default is base64",
-        default="base64",
+    CHILD_CHUNKS_PREVIEW_NUMBER: PositiveInt = Field(
+        description="Maximum number of child chunks to preview",
+        default=50,
     )
 
-    MULTIMODAL_SEND_VIDEO_FORMAT: Literal["base64", "url"] = Field(
-        description="Format for sending videos in multimodal contexts ('base64' or 'url'), default is base64",
+
+class MultiModalTransferConfig(BaseSettings):
+    MULTIMODAL_SEND_FORMAT: Literal["base64", "url"] = Field(
+        description="Format for sending files in multimodal contexts ('base64' or 'url'), default is base64",
         default="base64",
     )
 
@@ -710,27 +781,27 @@ class PositionConfig(BaseSettings):
         default="",
     )
 
-    @computed_field
+    @property
     def POSITION_PROVIDER_PINS_LIST(self) -> list[str]:
         return [item.strip() for item in self.POSITION_PROVIDER_PINS.split(",") if item.strip() != ""]
 
-    @computed_field
+    @property
     def POSITION_PROVIDER_INCLUDES_SET(self) -> set[str]:
         return {item.strip() for item in self.POSITION_PROVIDER_INCLUDES.split(",") if item.strip() != ""}
 
-    @computed_field
+    @property
     def POSITION_PROVIDER_EXCLUDES_SET(self) -> set[str]:
         return {item.strip() for item in self.POSITION_PROVIDER_EXCLUDES.split(",") if item.strip() != ""}
 
-    @computed_field
+    @property
     def POSITION_TOOL_PINS_LIST(self) -> list[str]:
         return [item.strip() for item in self.POSITION_TOOL_PINS.split(",") if item.strip() != ""]
 
-    @computed_field
+    @property
     def POSITION_TOOL_INCLUDES_SET(self) -> set[str]:
         return {item.strip() for item in self.POSITION_TOOL_INCLUDES.split(",") if item.strip() != ""}
 
-    @computed_field
+    @property
     def POSITION_TOOL_EXCLUDES_SET(self) -> set[str]:
         return {item.strip() for item in self.POSITION_TOOL_EXCLUDES.split(",") if item.strip() != ""}
 
@@ -762,24 +833,33 @@ class LoginConfig(BaseSettings):
     )
 
 
+class AccountConfig(BaseSettings):
+    ACCOUNT_DELETION_TOKEN_EXPIRY_MINUTES: PositiveInt = Field(
+        description="Duration in minutes for which a account deletion token remains valid",
+        default=5,
+    )
+
+
 class FeatureConfig(
     # place the configs in alphabet order
     AppExecutionConfig,
     AuthConfig,  # Changed from OAuthConfig to AuthConfig
     BillingConfig,
     CodeExecutionSandboxConfig,
+    PluginConfig,
+    MarketplaceConfig,
     DataSetConfig,
     EndpointConfig,
     FileAccessConfig,
     FileUploadConfig,
     HttpConfig,
-    VisionFormatConfig,
     InnerAPIConfig,
     IndexingConfig,
     LoggingConfig,
     MailConfig,
     ModelLoadBalanceConfig,
     ModerationConfig,
+    MultiModalTransferConfig,
     PositionConfig,
     RagEtlConfig,
     SecurityConfig,
@@ -789,6 +869,7 @@ class FeatureConfig(
     WorkflowNodeExecutionConfig,
     WorkspaceConfig,
     LoginConfig,
+    AccountConfig,
     # hosted services config
     HostedServiceConfig,
     CeleryBeatConfig,

api/configs/middleware/__init__.py

@@ -130,7 +130,6 @@ class DatabaseConfig(BaseSettings):
     )
 
     @computed_field
-    @property
     def SQLALCHEMY_DATABASE_URI(self) -> str:
         db_extras = (
             f"{self.DB_EXTRAS}&client_encoding={self.DB_CHARSET}" if self.DB_CHARSET else self.DB_EXTRAS
@@ -168,7 +167,6 @@ class DatabaseConfig(BaseSettings):
     )
 
     @computed_field
-    @property
     def SQLALCHEMY_ENGINE_OPTIONS(self) -> dict[str, Any]:
         return {
             "pool_size": self.SQLALCHEMY_POOL_SIZE,
@@ -206,7 +204,6 @@ class CeleryConfig(DatabaseConfig):
     )
 
     @computed_field
-    @property
     def CELERY_RESULT_BACKEND(self) -> str | None:
         return (
             "db+{}".format(self.SQLALCHEMY_DATABASE_URI)
@@ -214,7 +211,6 @@ class CeleryConfig(DatabaseConfig):
             else self.CELERY_BROKER_URL
         )
 
-    @computed_field
     @property
     def BROKER_USE_SSL(self) -> bool:
         return self.CELERY_BROKER_URL.startswith("rediss://") if self.CELERY_BROKER_URL else False

api/configs/middleware/storage/opendal_storage_config.py

@@ -1,51 +1,9 @@
-from enum import StrEnum
-from typing import Literal
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
 
-class OpenDALScheme(StrEnum):
-    FS = "fs"
-    S3 = "s3"
-
-
 class OpenDALStorageConfig(BaseSettings):
-    STORAGE_OPENDAL_SCHEME: str = Field(
-        default=OpenDALScheme.FS.value,
+    OPENDAL_SCHEME: str = Field(
+        default="fs",
         description="OpenDAL scheme.",
     )
-    # FS
-    OPENDAL_FS_ROOT: str = Field(
-        default="storage",
-        description="Root path for local storage.",
-    )
-    # S3
-    OPENDAL_S3_ROOT: str = Field(
-        default="/",
-        description="Root path for S3 storage.",
-    )
-    OPENDAL_S3_BUCKET: str = Field(
-        default="",
-        description="S3 bucket name.",
-    )
-    OPENDAL_S3_ENDPOINT: str = Field(
-        default="https://s3.amazonaws.com",
-        description="S3 endpoint URL.",
-    )
-    OPENDAL_S3_ACCESS_KEY_ID: str = Field(
-        default="",
-        description="S3 access key ID.",
-    )
-    OPENDAL_S3_SECRET_ACCESS_KEY: str = Field(
-        default="",
-        description="S3 secret access key.",
-    )
-    OPENDAL_S3_REGION: str = Field(
-        default="",
-        description="S3 region.",
-    )
-    OPENDAL_S3_SERVER_SIDE_ENCRYPTION: Literal["aws:kms", ""] = Field(
-        default="",
-        description="S3 server-side encryption.",
-    )

api/configs/middleware/vdb/milvus_config.py

@@ -33,3 +33,9 @@ class MilvusConfig(BaseSettings):
         description="Name of the Milvus database to connect to (default is 'default')",
         default="default",
     )
+
+    MILVUS_ENABLE_HYBRID_SEARCH: bool = Field(
+        description="Enable hybrid search features (requires Milvus >= 2.5.0). Set to false for compatibility with "
+        "older versions",
+        default=True,
+    )

api/configs/packaging/__init__.py

@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
 
     CURRENT_VERSION: str = Field(
         description="Dify version",
-        default="0.14.0",
+        default="0.15.0",
     )
 
     COMMIT_SHA: str = Field(

api/configs/remote_settings_sources/apollo/client.py

@@ -4,6 +4,7 @@ import logging
 import os
 import threading
 import time
+from collections.abc import Mapping
 from pathlib import Path
 
 from .python_3x import http_request, makedirs_wrapper
@@ -255,8 +256,8 @@ class ApolloClient:
         logger.info("stopped, long_poll")
 
     # add the need for endorsement to the header
-    def _sign_headers(self, url):
-        headers = {}
+    def _sign_headers(self, url: str) -> Mapping[str, str]:
+        headers: dict[str, str] = {}
         if self.secret == "":
             return headers
         uri = url[len(self.config_url) : len(url)]

api/constants/model_template.py

@@ -1,8 +1,9 @@
 import json
+from collections.abc import Mapping
 
 from models.model import AppMode
 
-default_app_templates = {
+default_app_templates: Mapping[AppMode, Mapping] = {
     # workflow default mode
     AppMode.WORKFLOW: {
         "app": {

api/contexts/__init__.py

@@ -1,9 +1,19 @@
 from contextvars import ContextVar
+from threading import Lock
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
+    from core.plugin.entities.plugin_daemon import PluginModelProviderEntity
+    from core.tools.plugin_tool.provider import PluginToolProviderController
     from core.workflow.entities.variable_pool import VariablePool
 
+
 tenant_id: ContextVar[str] = ContextVar("tenant_id")
 
 workflow_variable_pool: ContextVar["VariablePool"] = ContextVar("workflow_variable_pool")
+
+plugin_tool_providers: ContextVar[dict[str, "PluginToolProviderController"]] = ContextVar("plugin_tool_providers")
+plugin_tool_providers_lock: ContextVar[Lock] = ContextVar("plugin_tool_providers_lock")
+
+plugin_model_providers: ContextVar[list["PluginModelProviderEntity"] | None] = ContextVar("plugin_model_providers")
+plugin_model_providers_lock: ContextVar[Lock] = ContextVar("plugin_model_providers_lock")

api/controllers/common/errors.py

@@ -4,3 +4,8 @@ from werkzeug.exceptions import HTTPException
 class FilenameNotExistsError(HTTPException):
     code = 400
     description = "The specified filename does not exist."
+
+
+class RemoteFileUploadError(HTTPException):
+    code = 400
+    description = "Error uploading remote file."

api/controllers/common/fields.py

@@ -1,4 +1,4 @@
-from flask_restful import fields
+from flask_restful import fields  # type: ignore
 
 parameters__system_parameters = {
     "image_file_size_limit": fields.Integer,

api/controllers/console/__init__.py

@@ -2,7 +2,26 @@ from flask import Blueprint
 
 from libs.external_api import ExternalApi
 
-from .app.app_import import AppImportApi, AppImportConfirmApi
+from .app.app_import import AppImportApi, AppImportCheckDependenciesApi, AppImportConfirmApi
+from .explore.audio import ChatAudioApi, ChatTextApi
+from .explore.completion import ChatApi, ChatStopApi, CompletionApi, CompletionStopApi
+from .explore.conversation import (
+    ConversationApi,
+    ConversationListApi,
+    ConversationPinApi,
+    ConversationRenameApi,
+    ConversationUnPinApi,
+)
+from .explore.message import (
+    MessageFeedbackApi,
+    MessageListApi,
+    MessageMoreLikeThisApi,
+    MessageSuggestedQuestionApi,
+)
+from .explore.workflow import (
+    InstalledAppWorkflowRunApi,
+    InstalledAppWorkflowTaskStopApi,
+)
 from .files import FileApi, FilePreviewApi, FileSupportTypeApi
 from .remote_files import RemoteFileInfoApi, RemoteFileUploadApi
 
@@ -21,6 +40,7 @@ api.add_resource(RemoteFileUploadApi, "/remote-files/upload")
 # Import App
 api.add_resource(AppImportApi, "/apps/imports")
 api.add_resource(AppImportConfirmApi, "/apps/imports/<string:import_id>/confirm")
+api.add_resource(AppImportCheckDependenciesApi, "/apps/imports/<string:app_id>/check-dependencies")
 
 # Import other controllers
 from . import admin, apikey, extension, feature, ping, setup, version
@@ -66,19 +86,96 @@ from .datasets import (
 
 # Import explore controllers
 from .explore import (
-    audio,
-    completion,
-    conversation,
     installed_app,
-    message,
     parameter,
     recommended_app,
     saved_message,
-    workflow,
+)
+
+# Explore Audio
+api.add_resource(ChatAudioApi, "/installed-apps/<uuid:installed_app_id>/audio-to-text", endpoint="installed_app_audio")
+api.add_resource(ChatTextApi, "/installed-apps/<uuid:installed_app_id>/text-to-audio", endpoint="installed_app_text")
+
+# Explore Completion
+api.add_resource(
+    CompletionApi, "/installed-apps/<uuid:installed_app_id>/completion-messages", endpoint="installed_app_completion"
+)
+api.add_resource(
+    CompletionStopApi,
+    "/installed-apps/<uuid:installed_app_id>/completion-messages/<string:task_id>/stop",
+    endpoint="installed_app_stop_completion",
+)
+api.add_resource(
+    ChatApi, "/installed-apps/<uuid:installed_app_id>/chat-messages", endpoint="installed_app_chat_completion"
+)
+api.add_resource(
+    ChatStopApi,
+    "/installed-apps/<uuid:installed_app_id>/chat-messages/<string:task_id>/stop",
+    endpoint="installed_app_stop_chat_completion",
+)
+
+# Explore Conversation
+api.add_resource(
+    ConversationRenameApi,
+    "/installed-apps/<uuid:installed_app_id>/conversations/<uuid:c_id>/name",
+    endpoint="installed_app_conversation_rename",
+)
+api.add_resource(
+    ConversationListApi, "/installed-apps/<uuid:installed_app_id>/conversations", endpoint="installed_app_conversations"
+)
+api.add_resource(
+    ConversationApi,
+    "/installed-apps/<uuid:installed_app_id>/conversations/<uuid:c_id>",
+    endpoint="installed_app_conversation",
+)
+api.add_resource(
+    ConversationPinApi,
+    "/installed-apps/<uuid:installed_app_id>/conversations/<uuid:c_id>/pin",
+    endpoint="installed_app_conversation_pin",
+)
+api.add_resource(
+    ConversationUnPinApi,
+    "/installed-apps/<uuid:installed_app_id>/conversations/<uuid:c_id>/unpin",
+    endpoint="installed_app_conversation_unpin",
+)
+
+
+# Explore Message
+api.add_resource(MessageListApi, "/installed-apps/<uuid:installed_app_id>/messages", endpoint="installed_app_messages")
+api.add_resource(
+    MessageFeedbackApi,
+    "/installed-apps/<uuid:installed_app_id>/messages/<uuid:message_id>/feedbacks",
+    endpoint="installed_app_message_feedback",
+)
+api.add_resource(
+    MessageMoreLikeThisApi,
+    "/installed-apps/<uuid:installed_app_id>/messages/<uuid:message_id>/more-like-this",
+    endpoint="installed_app_more_like_this",
+)
+api.add_resource(
+    MessageSuggestedQuestionApi,
+    "/installed-apps/<uuid:installed_app_id>/messages/<uuid:message_id>/suggested-questions",
+    endpoint="installed_app_suggested_question",
+)
+# Explore Workflow
+api.add_resource(InstalledAppWorkflowRunApi, "/installed-apps/<uuid:installed_app_id>/workflows/run")
+api.add_resource(
+    InstalledAppWorkflowTaskStopApi, "/installed-apps/<uuid:installed_app_id>/workflows/tasks/<string:task_id>/stop"
 )
 
 # Import tag controllers
 from .tag import tags
 
 # Import workspace controllers
-from .workspace import account, load_balancing_config, members, model_providers, models, tool_providers, workspace
+from .workspace import (
+    account,
+    agent_providers,
+    endpoint,
+    load_balancing_config,
+    members,
+    model_providers,
+    models,
+    plugin,
+    tool_providers,
+    workspace,
+)

api/controllers/console/admin.py

@@ -1,7 +1,9 @@
 from functools import wraps
 
 from flask import request
-from flask_restful import Resource, reqparse
+from flask_restful import Resource, reqparse  # type: ignore
+from sqlalchemy import select
+from sqlalchemy.orm import Session
 from werkzeug.exceptions import NotFound, Unauthorized
 
 from configs import dify_config
@@ -31,7 +33,7 @@ def admin_required(view):
         if auth_scheme != "bearer":
             raise Unauthorized("Invalid Authorization header format. Expected 'Bearer <api-key>' format.")
 
-        if dify_config.ADMIN_API_KEY != auth_token:
+        if auth_token != dify_config.ADMIN_API_KEY:
             raise Unauthorized("API key is invalid.")
 
         return view(*args, **kwargs)
@@ -54,7 +56,8 @@ class InsertExploreAppListApi(Resource):
         parser.add_argument("position", type=int, required=True, nullable=False, location="json")
         args = parser.parse_args()
 
-        app = App.query.filter(App.id == args["app_id"]).first()
+        with Session(db.engine) as session:
+            app = session.execute(select(App).filter(App.id == args["app_id"])).scalar_one_or_none()
         if not app:
             raise NotFound(f'App \'{args["app_id"]}\' is not found')
 
@@ -70,7 +73,10 @@ class InsertExploreAppListApi(Resource):
             privacy_policy = site.privacy_policy or args["privacy_policy"] or ""
             custom_disclaimer = site.custom_disclaimer or args["custom_disclaimer"] or ""
 
-        recommended_app = RecommendedApp.query.filter(RecommendedApp.app_id == args["app_id"]).first()
+        with Session(db.engine) as session:
+            recommended_app = session.execute(
+                select(RecommendedApp).filter(RecommendedApp.app_id == args["app_id"])
+            ).scalar_one_or_none()
 
         if not recommended_app:
             recommended_app = RecommendedApp(
@@ -110,17 +116,27 @@ class InsertExploreAppApi(Resource):
     @only_edition_cloud
     @admin_required
     def delete(self, app_id):
-        recommended_app = RecommendedApp.query.filter(RecommendedApp.app_id == str(app_id)).first()
+        with Session(db.engine) as session:
+            recommended_app = session.execute(
+                select(RecommendedApp).filter(RecommendedApp.app_id == str(app_id))
+            ).scalar_one_or_none()
+
         if not recommended_app:
             return {"result": "success"}, 204
 
-        app = App.query.filter(App.id == recommended_app.app_id).first()
+        with Session(db.engine) as session:
+            app = session.execute(select(App).filter(App.id == recommended_app.app_id)).scalar_one_or_none()
+
         if app:
             app.is_public = False
 
-        installed_apps = InstalledApp.query.filter(
-            InstalledApp.app_id == recommended_app.app_id, InstalledApp.tenant_id != InstalledApp.app_owner_tenant_id
-        ).all()
+        with Session(db.engine) as session:
+            installed_apps = session.execute(
+                select(InstalledApp).filter(
+                    InstalledApp.app_id == recommended_app.app_id,
+                    InstalledApp.tenant_id != InstalledApp.app_owner_tenant_id,
+                )
+            ).all()
 
         for installed_app in installed_apps:
             db.session.delete(installed_app)

api/controllers/console/apikey.py

@@ -1,6 +1,10 @@
-import flask_restful
-from flask_login import current_user
+from typing import Any
+
+import flask_restful  # type: ignore
+from flask_login import current_user  # type: ignore
 from flask_restful import Resource, fields, marshal_with
+from sqlalchemy import select
+from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden
 
 from extensions.ext_database import db
@@ -24,7 +28,16 @@ api_key_list = {"data": fields.List(fields.Nested(api_key_fields), attribute="it
 
 
 def _get_resource(resource_id, tenant_id, resource_model):
-    resource = resource_model.query.filter_by(id=resource_id, tenant_id=tenant_id).first()
+    if resource_model == App:
+        with Session(db.engine) as session:
+            resource = session.execute(
+                select(resource_model).filter_by(id=resource_id, tenant_id=tenant_id)
+            ).scalar_one_or_none()
+    else:
+        with Session(db.engine) as session:
+            resource = session.execute(
+                select(resource_model).filter_by(id=resource_id, tenant_id=tenant_id)
+            ).scalar_one_or_none()
 
     if resource is None:
         flask_restful.abort(404, message=f"{resource_model.__name__} not found.")
@@ -35,14 +48,15 @@ def _get_resource(resource_id, tenant_id, resource_model):
 class BaseApiKeyListResource(Resource):
     method_decorators = [account_initialization_required, login_required, setup_required]
 
-    resource_type = None
-    resource_model = None
-    resource_id_field = None
-    token_prefix = None
+    resource_type: str | None = None
+    resource_model: Any = None
+    resource_id_field: str | None = None
+    token_prefix: str | None = None
     max_keys = 10
 
     @marshal_with(api_key_list)
     def get(self, resource_id):
+        assert self.resource_id_field is not None, "resource_id_field must be set"
         resource_id = str(resource_id)
         _get_resource(resource_id, current_user.current_tenant_id, self.resource_model)
         keys = (
@@ -54,6 +68,7 @@ class BaseApiKeyListResource(Resource):
 
     @marshal_with(api_key_fields)
     def post(self, resource_id):
+        assert self.resource_id_field is not None, "resource_id_field must be set"
         resource_id = str(resource_id)
         _get_resource(resource_id, current_user.current_tenant_id, self.resource_model)
         if not current_user.is_editor:
@@ -86,11 +101,12 @@ class BaseApiKeyListResource(Resource):
 class BaseApiKeyResource(Resource):
     method_decorators = [account_initialization_required, login_required, setup_required]
 
-    resource_type = None
-    resource_model = None
-    resource_id_field = None
+    resource_type: str | None = None
+    resource_model: Any = None
+    resource_id_field: str | None = None
 
     def delete(self, resource_id, api_key_id):
+        assert self.resource_id_field is not None, "resource_id_field must be set"
         resource_id = str(resource_id)
         api_key_id = str(api_key_id)
         _get_resource(resource_id, current_user.current_tenant_id, self.resource_model)

api/controllers/console/app/advanced_prompt_template.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource, reqparse
+from flask_restful import Resource, reqparse  # type: ignore
 
 from controllers.console import api
 from controllers.console.wraps import account_initialization_required, setup_required

api/controllers/console/app/agent.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource, reqparse
+from flask_restful import Resource, reqparse  # type: ignore
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model

api/controllers/console/app/annotation.py

@@ -1,6 +1,6 @@
 from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal, marshal_with, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, marshal, marshal_with, reqparse  # type: ignore
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
@@ -110,7 +110,7 @@ class AnnotationListApi(Resource):
 
         page = request.args.get("page", default=1, type=int)
         limit = request.args.get("limit", default=20, type=int)
-        keyword = request.args.get("keyword", default=None, type=str)
+        keyword = request.args.get("keyword", default="", type=str)
 
         app_id = str(app_id)
         annotation_list, total = AppAnnotationService.get_annotation_list_by_app_id(app_id, page, limit, keyword)

api/controllers/console/app/app.py

@@ -1,8 +1,8 @@
 import uuid
 from typing import cast
 
-from flask_login import current_user
-from flask_restful import Resource, inputs, marshal, marshal_with, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, inputs, marshal, marshal_with, reqparse  # type: ignore
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import BadRequest, Forbidden, abort
@@ -57,12 +57,13 @@ class AppListApi(Resource):
         )
         parser.add_argument("name", type=str, location="args", required=False)
         parser.add_argument("tag_ids", type=uuid_list, location="args", required=False)
+        parser.add_argument("is_created_by_me", type=inputs.boolean, location="args", required=False)
 
         args = parser.parse_args()
 
         # get app list
         app_service = AppService()
-        app_pagination = app_service.get_paginate_apps(current_user.current_tenant_id, args)
+        app_pagination = app_service.get_paginate_apps(current_user.id, current_user.current_tenant_id, args)
         if not app_pagination:
             return {"data": [], "total": 0, "page": 1, "limit": 20, "has_more": False}
 

api/controllers/console/app/app_import.py

@@ -1,18 +1,20 @@
 from typing import cast
 
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse  # type: ignore
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden
 
+from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import (
     account_initialization_required,
     setup_required,
 )
 from extensions.ext_database import db
-from fields.app_fields import app_import_fields
+from fields.app_fields import app_import_check_dependencies_fields, app_import_fields
 from libs.login import login_required
 from models import Account
+from models.model import App
 from services.app_dsl_service import AppDslService, ImportStatus
 
 
@@ -88,3 +90,20 @@ class AppImportConfirmApi(Resource):
         if result.status == ImportStatus.FAILED.value:
             return result.model_dump(mode="json"), 400
         return result.model_dump(mode="json"), 200
+
+
+class AppImportCheckDependenciesApi(Resource):
+    @setup_required
+    @login_required
+    @get_app_model
+    @account_initialization_required
+    @marshal_with(app_import_check_dependencies_fields)
+    def get(self, app_model: App):
+        if not current_user.is_editor:
+            raise Forbidden()
+
+        with Session(db.engine) as session:
+            import_service = AppDslService(session)
+            result = import_service.check_dependencies(app_model=app_model)
+
+        return result.model_dump(mode="json"), 200

api/controllers/console/app/audio.py

@@ -1,7 +1,7 @@
 import logging
 
 from flask import request
-from flask_restful import Resource, reqparse
+from flask_restful import Resource, reqparse  # type: ignore
 from werkzeug.exceptions import InternalServerError
 
 import services

api/controllers/console/app/completion.py

@@ -1,7 +1,7 @@
 import logging
 
-import flask_login
-from flask_restful import Resource, reqparse
+import flask_login  # type: ignore
+from flask_restful import Resource, reqparse  # type: ignore
 from werkzeug.exceptions import InternalServerError, NotFound
 
 import services
@@ -20,7 +20,6 @@ from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpErr
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom
 from core.errors.error import (
-    AppInvokeQuotaExceededError,
     ModelCurrentlyNotSupportError,
     ProviderTokenNotInitError,
     QuotaExceededError,
@@ -76,7 +75,7 @@ class CompletionMessageApi(Resource):
             raise ProviderModelCurrentlyNotSupportError()
         except InvokeError as e:
             raise CompletionRequestError(e.description)
-        except (ValueError, AppInvokeQuotaExceededError) as e:
+        except ValueError as e:
             raise e
         except Exception as e:
             logging.exception("internal server error.")
@@ -141,7 +140,7 @@ class ChatMessageApi(Resource):
             raise InvokeRateLimitHttpError(ex.description)
         except InvokeError as e:
             raise CompletionRequestError(e.description)
-        except (ValueError, AppInvokeQuotaExceededError) as e:
+        except ValueError as e:
             raise e
         except Exception as e:
             logging.exception("internal server error.")

api/controllers/console/app/conversation.py

@@ -1,9 +1,9 @@
 from datetime import UTC, datetime
 
-import pytz
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
-from flask_restful.inputs import int_range
+import pytz  # pip install pytz
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from flask_restful.inputs import int_range  # type: ignore
 from sqlalchemy import func, or_
 from sqlalchemy.orm import joinedload
 from werkzeug.exceptions import Forbidden, NotFound
@@ -77,8 +77,9 @@ class CompletionConversationApi(Resource):
 
             query = query.where(Conversation.created_at < end_datetime_utc)
 
+        # FIXME, the type ignore in this file
         if args["annotation_status"] == "annotated":
-            query = query.options(joinedload(Conversation.message_annotations)).join(
+            query = query.options(joinedload(Conversation.message_annotations)).join(  # type: ignore
                 MessageAnnotation, MessageAnnotation.conversation_id == Conversation.id
             )
         elif args["annotation_status"] == "not_annotated":
@@ -222,7 +223,7 @@ class ChatConversationApi(Resource):
                     query = query.where(Conversation.created_at <= end_datetime_utc)
 
         if args["annotation_status"] == "annotated":
-            query = query.options(joinedload(Conversation.message_annotations)).join(
+            query = query.options(joinedload(Conversation.message_annotations)).join(  # type: ignore
                 MessageAnnotation, MessageAnnotation.conversation_id == Conversation.id
             )
         elif args["annotation_status"] == "not_annotated":
@@ -234,7 +235,7 @@ class ChatConversationApi(Resource):
 
         if args["message_count_gte"] and args["message_count_gte"] >= 1:
             query = (
-                query.options(joinedload(Conversation.messages))
+                query.options(joinedload(Conversation.messages))  # type: ignore
                 .join(Message, Message.conversation_id == Conversation.id)
                 .group_by(Conversation.id)
                 .having(func.count(Message.id) >= args["message_count_gte"])

api/controllers/console/app/conversation_variables.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource, marshal_with, reqparse
+from flask_restful import Resource, marshal_with, reqparse  # type: ignore
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 

api/controllers/console/app/generator.py

@@ -1,7 +1,7 @@
 import os
 
-from flask_login import current_user
-from flask_restful import Resource, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, reqparse  # type: ignore
 
 from controllers.console import api
 from controllers.console.app.error import (

api/controllers/console/app/message.py

@@ -1,8 +1,8 @@
 import logging
 
-from flask_login import current_user
-from flask_restful import Resource, fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, fields, marshal_with, reqparse  # type: ignore
+from flask_restful.inputs import int_range  # type: ignore
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 from controllers.console import api

api/controllers/console/app/model_config.py

@@ -1,8 +1,9 @@
 import json
+from typing import cast
 
 from flask import request
-from flask_login import current_user
-from flask_restful import Resource
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource  # type: ignore
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
@@ -26,7 +27,9 @@ class ModelConfigResource(Resource):
         """Modify app model config"""
         # validate config
         model_configuration = AppModelConfigService.validate_configuration(
-            tenant_id=current_user.current_tenant_id, config=request.json, app_mode=AppMode.value_of(app_model.mode)
+            tenant_id=current_user.current_tenant_id,
+            config=cast(dict, request.json),
+            app_mode=AppMode.value_of(app_model.mode),
         )
 
         new_app_model_config = AppModelConfig(
@@ -38,9 +41,11 @@ class ModelConfigResource(Resource):
 
         if app_model.mode == AppMode.AGENT_CHAT.value or app_model.is_agent:
             # get original app model config
-            original_app_model_config: AppModelConfig = (
+            original_app_model_config = (
                 db.session.query(AppModelConfig).filter(AppModelConfig.id == app_model.app_model_config_id).first()
             )
+            if original_app_model_config is None:
+                raise ValueError("Original app model config not found")
             agent_mode = original_app_model_config.agent_mode_dict
             # decrypt agent tool parameters if it's secret-input
             parameter_map = {}
@@ -65,7 +70,7 @@ class ModelConfigResource(Resource):
                         provider_type=agent_tool_entity.provider_type,
                         identity_id=f"AGENT.{app_model.id}",
                     )
-                except Exception as e:
+                except Exception:
                     continue
 
                 # get decrypted parameters
@@ -97,7 +102,7 @@ class ModelConfigResource(Resource):
                             app_id=app_model.id,
                             agent_tool=agent_tool_entity,
                         )
-                    except Exception as e:
+                    except Exception:
                         continue
 
                 manager = ToolParameterConfigurationManager(

api/controllers/console/app/ops_trace.py

@@ -1,4 +1,5 @@
-from flask_restful import Resource, reqparse
+from flask_restful import Resource, reqparse  # type: ignore
+from werkzeug.exceptions import BadRequest
 
 from controllers.console import api
 from controllers.console.app.error import TracingConfigCheckError, TracingConfigIsExist, TracingConfigNotExist
@@ -26,7 +27,7 @@ class TraceAppConfigApi(Resource):
                 return {"has_not_configured": True}
             return trace_config
         except Exception as e:
-            raise e
+            raise BadRequest(str(e))
 
     @setup_required
     @login_required
@@ -48,7 +49,7 @@ class TraceAppConfigApi(Resource):
                 raise TracingConfigCheckError()
             return result
         except Exception as e:
-            raise e
+            raise BadRequest(str(e))
 
     @setup_required
     @login_required
@@ -68,7 +69,7 @@ class TraceAppConfigApi(Resource):
                 raise TracingConfigNotExist()
             return {"result": "success"}
         except Exception as e:
-            raise e
+            raise BadRequest(str(e))
 
     @setup_required
     @login_required
@@ -85,7 +86,7 @@ class TraceAppConfigApi(Resource):
                 raise TracingConfigNotExist()
             return {"result": "success"}
         except Exception as e:
-            raise e
+            raise BadRequest(str(e))
 
 
 api.add_resource(TraceAppConfigApi, "/apps/<uuid:app_id>/trace-config")

api/controllers/console/app/site.py

@@ -1,7 +1,8 @@
 from datetime import UTC, datetime
 
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden, NotFound
 
 from constants.languages import supported_language
@@ -50,33 +51,37 @@ class AppSite(Resource):
         if not current_user.is_editor:
             raise Forbidden()
 
-        site = db.session.query(Site).filter(Site.app_id == app_model.id).one_or_404()
+        with Session(db.engine) as session:
+            site = session.query(Site).filter(Site.app_id == app_model.id).first()
 
-        for attr_name in [
-            "title",
-            "icon_type",
-            "icon",
-            "icon_background",
-            "description",
-            "default_language",
-            "chat_color_theme",
-            "chat_color_theme_inverted",
-            "customize_domain",
-            "copyright",
-            "privacy_policy",
-            "custom_disclaimer",
-            "customize_token_strategy",
-            "prompt_public",
-            "show_workflow_steps",
-            "use_icon_as_answer_icon",
-        ]:
-            value = args.get(attr_name)
-            if value is not None:
-                setattr(site, attr_name, value)
+            if not site:
+                raise NotFound
 
-        site.updated_by = current_user.id
-        site.updated_at = datetime.now(UTC).replace(tzinfo=None)
-        db.session.commit()
+            for attr_name in [
+                "title",
+                "icon_type",
+                "icon",
+                "icon_background",
+                "description",
+                "default_language",
+                "chat_color_theme",
+                "chat_color_theme_inverted",
+                "customize_domain",
+                "copyright",
+                "privacy_policy",
+                "custom_disclaimer",
+                "customize_token_strategy",
+                "prompt_public",
+                "show_workflow_steps",
+                "use_icon_as_answer_icon",
+            ]:
+                value = args.get(attr_name)
+                if value is not None:
+                    setattr(site, attr_name, value)
+
+            site.updated_by = current_user.id
+            site.updated_at = datetime.now(UTC).replace(tzinfo=None)
+            session.commit()
 
         return site
 

api/controllers/console/app/statistic.py

@@ -3,8 +3,8 @@ from decimal import Decimal
 
 import pytz
 from flask import jsonify
-from flask_login import current_user
-from flask_restful import Resource, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, reqparse  # type: ignore
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
@@ -273,8 +273,7 @@ FROM
             messages m
             ON c.id = m.conversation_id
         WHERE
-            c.override_model_configs IS NULL
-            AND c.app_id = :app_id"""
+            c.app_id = :app_id"""
         arg_dict = {"tz": account.timezone, "app_id": app_model.id}
 
         timezone = pytz.timezone(account.timezone)

api/controllers/console/app/workflow.py

@@ -2,10 +2,11 @@ import json
 import logging
 
 from flask import abort, request
-from flask_restful import Resource, marshal_with, reqparse
+from flask_restful import Resource, inputs, marshal_with, reqparse  # type: ignore
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 import services
+from configs import dify_config
 from controllers.console import api
 from controllers.console.app.error import ConversationCompletedError, DraftWorkflowNotExist, DraftWorkflowNotSync
 from controllers.console.app.wraps import get_app_model
@@ -13,12 +14,13 @@ from controllers.console.wraps import account_initialization_required, setup_req
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom
 from factories import variable_factory
-from fields.workflow_fields import workflow_fields
+from fields.workflow_fields import workflow_fields, workflow_pagination_fields
 from fields.workflow_run_fields import workflow_run_node_execution_fields
 from libs import helper
 from libs.helper import TimestampField, uuid_value
 from libs.login import current_user, login_required
 from models import App
+from models.account import Account
 from models.model import AppMode
 from services.app_generate_service import AppGenerateService
 from services.errors.app import WorkflowHashNotEqualError
@@ -95,6 +97,9 @@ class DraftWorkflowApi(Resource):
         else:
             abort(415)
 
+        if not isinstance(current_user, Account):
+            raise Forbidden()
+
         workflow_service = WorkflowService()
 
         try:
@@ -138,6 +143,9 @@ class AdvancedChatDraftWorkflowRunApi(Resource):
         if not current_user.is_editor:
             raise Forbidden()
 
+        if not isinstance(current_user, Account):
+            raise Forbidden()
+
         parser = reqparse.RequestParser()
         parser.add_argument("inputs", type=dict, location="json")
         parser.add_argument("query", type=str, required=True, location="json", default="")
@@ -159,7 +167,7 @@ class AdvancedChatDraftWorkflowRunApi(Resource):
             raise ConversationCompletedError()
         except ValueError as e:
             raise e
-        except Exception as e:
+        except Exception:
             logging.exception("internal server error.")
             raise InternalServerError()
 
@@ -177,38 +185,7 @@ class AdvancedChatDraftRunIterationNodeApi(Resource):
         if not current_user.is_editor:
             raise Forbidden()
 
-        parser = reqparse.RequestParser()
-        parser.add_argument("inputs", type=dict, location="json")
-        args = parser.parse_args()
-
-        try:
-            response = AppGenerateService.generate_single_iteration(
-                app_model=app_model, user=current_user, node_id=node_id, args=args, streaming=True
-            )
-
-            return helper.compact_generate_response(response)
-        except services.errors.conversation.ConversationNotExistsError:
-            raise NotFound("Conversation Not Exists.")
-        except services.errors.conversation.ConversationCompletedError:
-            raise ConversationCompletedError()
-        except ValueError as e:
-            raise e
-        except Exception as e:
-            logging.exception("internal server error.")
-            raise InternalServerError()
-
-
-class WorkflowDraftRunIterationNodeApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @get_app_model(mode=[AppMode.WORKFLOW])
-    def post(self, app_model: App, node_id: str):
-        """
-        Run draft workflow iteration node
-        """
-        # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        if not isinstance(current_user, Account):
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -227,7 +204,44 @@ class WorkflowDraftRunIterationNodeApi(Resource):
             raise ConversationCompletedError()
         except ValueError as e:
             raise e
-        except Exception as e:
+        except Exception:
+            logging.exception("internal server error.")
+            raise InternalServerError()
+
+
+class WorkflowDraftRunIterationNodeApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.WORKFLOW])
+    def post(self, app_model: App, node_id: str):
+        """
+        Run draft workflow iteration node
+        """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+
+        if not isinstance(current_user, Account):
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("inputs", type=dict, location="json")
+        args = parser.parse_args()
+
+        try:
+            response = AppGenerateService.generate_single_iteration(
+                app_model=app_model, user=current_user, node_id=node_id, args=args, streaming=True
+            )
+
+            return helper.compact_generate_response(response)
+        except services.errors.conversation.ConversationNotExistsError:
+            raise NotFound("Conversation Not Exists.")
+        except services.errors.conversation.ConversationCompletedError:
+            raise ConversationCompletedError()
+        except ValueError as e:
+            raise e
+        except Exception:
             logging.exception("internal server error.")
             raise InternalServerError()
 
@@ -245,6 +259,9 @@ class DraftWorkflowRunApi(Resource):
         if not current_user.is_editor:
             raise Forbidden()
 
+        if not isinstance(current_user, Account):
+            raise Forbidden()
+
         parser = reqparse.RequestParser()
         parser.add_argument("inputs", type=dict, required=True, nullable=False, location="json")
         parser.add_argument("files", type=list, required=False, location="json")
@@ -293,13 +310,20 @@ class DraftWorkflowNodeRunApi(Resource):
         if not current_user.is_editor:
             raise Forbidden()
 
+        if not isinstance(current_user, Account):
+            raise Forbidden()
+
         parser = reqparse.RequestParser()
         parser.add_argument("inputs", type=dict, required=True, nullable=False, location="json")
         args = parser.parse_args()
 
+        inputs = args.get("inputs")
+        if inputs == None:
+            raise ValueError("missing inputs")
+
         workflow_service = WorkflowService()
         workflow_node_execution = workflow_service.run_draft_workflow_node(
-            app_model=app_model, node_id=node_id, user_inputs=args.get("inputs"), account=current_user
+            app_model=app_model, node_id=node_id, user_inputs=inputs, account=current_user
         )
 
         return workflow_node_execution
@@ -338,6 +362,9 @@ class PublishedWorkflowApi(Resource):
         if not current_user.is_editor:
             raise Forbidden()
 
+        if not isinstance(current_user, Account):
+            raise Forbidden()
+
         workflow_service = WorkflowService()
         workflow = workflow_service.publish_workflow(app_model=app_model, account=current_user)
 
@@ -375,12 +402,17 @@ class DefaultBlockConfigApi(Resource):
         if not current_user.is_editor:
             raise Forbidden()
 
+        if not isinstance(current_user, Account):
+            raise Forbidden()
+
         parser = reqparse.RequestParser()
         parser.add_argument("q", type=str, location="args")
         args = parser.parse_args()
 
+        q = args.get("q")
+
         filters = None
-        if args.get("q"):
+        if q:
             try:
                 filters = json.loads(args.get("q", ""))
             except json.JSONDecodeError:
@@ -406,6 +438,9 @@ class ConvertToWorkflowApi(Resource):
         if not current_user.is_editor:
             raise Forbidden()
 
+        if not isinstance(current_user, Account):
+            raise Forbidden()
+
         if request.data:
             parser = reqparse.RequestParser()
             parser.add_argument("name", type=str, required=False, nullable=True, location="json")
@@ -426,7 +461,46 @@ class ConvertToWorkflowApi(Resource):
         }
 
 
+class WorkflowConfigApi(Resource):
+    """Resource for workflow configuration."""
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    def get(self, app_model: App):
+        return {
+            "parallel_depth_limit": dify_config.WORKFLOW_PARALLEL_DEPTH_LIMIT,
+        }
+
+
+class PublishedAllWorkflowApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    @marshal_with(workflow_pagination_fields)
+    def get(self, app_model: App):
+        """
+        Get published workflows
+        """
+        if not current_user.is_editor:
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("page", type=inputs.int_range(1, 99999), required=False, default=1, location="args")
+        parser.add_argument("limit", type=inputs.int_range(1, 100), required=False, default=20, location="args")
+        args = parser.parse_args()
+        page = args.get("page")
+        limit = args.get("limit")
+        workflow_service = WorkflowService()
+        workflows, has_more = workflow_service.get_all_published_workflow(app_model=app_model, page=page, limit=limit)
+
+        return {"items": workflows, "page": page, "limit": limit, "has_more": has_more}
+
+
 api.add_resource(DraftWorkflowApi, "/apps/<uuid:app_id>/workflows/draft")
+api.add_resource(WorkflowConfigApi, "/apps/<uuid:app_id>/workflows/draft/config")
 api.add_resource(AdvancedChatDraftWorkflowRunApi, "/apps/<uuid:app_id>/advanced-chat/workflows/draft/run")
 api.add_resource(DraftWorkflowRunApi, "/apps/<uuid:app_id>/workflows/draft/run")
 api.add_resource(WorkflowTaskStopApi, "/apps/<uuid:app_id>/workflow-runs/tasks/<string:task_id>/stop")
@@ -439,6 +513,7 @@ api.add_resource(
     WorkflowDraftRunIterationNodeApi, "/apps/<uuid:app_id>/workflows/draft/iteration/nodes/<string:node_id>/run"
 )
 api.add_resource(PublishedWorkflowApi, "/apps/<uuid:app_id>/workflows/publish")
+api.add_resource(PublishedAllWorkflowApi, "/apps/<uuid:app_id>/workflows")
 api.add_resource(DefaultBlockConfigsApi, "/apps/<uuid:app_id>/workflows/default-workflow-block-configs")
 api.add_resource(
     DefaultBlockConfigApi, "/apps/<uuid:app_id>/workflows/default-workflow-block-configs/<string:block_type>"

api/controllers/console/app/workflow_app_log.py

@@ -1,5 +1,5 @@
-from flask_restful import Resource, marshal_with, reqparse
-from flask_restful.inputs import int_range
+from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from flask_restful.inputs import int_range  # type: ignore
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model

api/controllers/console/app/workflow_run.py

@@ -1,5 +1,5 @@
-from flask_restful import Resource, marshal_with, reqparse
-from flask_restful.inputs import int_range
+from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from flask_restful.inputs import int_range  # type: ignore
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model

api/controllers/console/app/workflow_statistic.py

@@ -3,8 +3,8 @@ from decimal import Decimal
 
 import pytz
 from flask import jsonify
-from flask_login import current_user
-from flask_restful import Resource, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, reqparse  # type: ignore
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model

api/controllers/console/app/wraps.py

@@ -5,11 +5,10 @@ from typing import Optional, Union
 from controllers.console.app.error import AppNotFoundError
 from extensions.ext_database import db
 from libs.login import current_user
-from models import App
-from models.model import AppMode
+from models import App, AppMode
 
 
-def get_app_model(view: Optional[Callable] = None, *, mode: Union[AppMode, list[AppMode]] = None):
+def get_app_model(view: Optional[Callable] = None, *, mode: Union[AppMode, list[AppMode], None] = None):
     def decorator(view_func):
         @wraps(view_func)
         def decorated_view(*args, **kwargs):

api/controllers/console/auth/activate.py

@@ -1,14 +1,14 @@
 import datetime
 
 from flask import request
-from flask_restful import Resource, reqparse
+from flask_restful import Resource, reqparse  # type: ignore
 
 from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.error import AlreadyActivateError
 from extensions.ext_database import db
 from libs.helper import StrLen, email, extract_remote_ip, timezone
-from models.account import AccountStatus, Tenant
+from models.account import AccountStatus
 from services.account_service import AccountService, RegisterService
 
 
@@ -27,7 +27,7 @@ class ActivateCheckApi(Resource):
         invitation = RegisterService.get_invitation_if_token_valid(workspaceId, reg_email, token)
         if invitation:
             data = invitation.get("data", {})
-            tenant: Tenant = invitation.get("tenant", None)
+            tenant = invitation.get("tenant", None)
             workspace_name = tenant.name if tenant else None
             workspace_id = tenant.id if tenant else None
             invitee_email = data.get("email") if data else None

api/controllers/console/auth/data_source_bearer_auth.py

@@ -1,5 +1,5 @@
-from flask_login import current_user
-from flask_restful import Resource, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, reqparse  # type: ignore
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api

api/controllers/console/auth/data_source_oauth.py

@@ -2,8 +2,8 @@ import logging
 
 import requests
 from flask import current_app, redirect, request
-from flask_login import current_user
-from flask_restful import Resource
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource  # type: ignore
 from werkzeug.exceptions import Forbidden
 
 from configs import dify_config
@@ -17,8 +17,8 @@ from ..wraps import account_initialization_required, setup_required
 def get_oauth_providers():
     with current_app.app_context():
         notion_oauth = NotionOAuth(
-            client_id=dify_config.NOTION_CLIENT_ID,
-            client_secret=dify_config.NOTION_CLIENT_SECRET,
+            client_id=dify_config.NOTION_CLIENT_ID or "",
+            client_secret=dify_config.NOTION_CLIENT_SECRET or "",
             redirect_uri=dify_config.CONSOLE_API_URL + "/console/api/oauth/data-source/callback/notion",
         )
 

api/controllers/console/auth/error.py

@@ -53,3 +53,9 @@ class EmailCodeLoginRateLimitExceededError(BaseHTTPException):
     error_code = "email_code_login_rate_limit_exceeded"
     description = "Too many login emails have been sent. Please try again in 5 minutes."
     code = 429
+
+
+class EmailCodeAccountDeletionRateLimitExceededError(BaseHTTPException):
+    error_code = "email_code_account_deletion_rate_limit_exceeded"
+    description = "Too many account deletion emails have been sent. Please try again in 5 minutes."
+    code = 429

api/controllers/console/auth/forgot_password.py

@@ -2,17 +2,14 @@ import base64
 import secrets
 
 from flask import request
-from flask_restful import Resource, reqparse
+from flask_restful import Resource, reqparse  # type: ignore
+from sqlalchemy import select
+from sqlalchemy.orm import Session
 
 from constants.languages import languages
 from controllers.console import api
-from controllers.console.auth.error import (
-    EmailCodeError,
-    InvalidEmailError,
-    InvalidTokenError,
-    PasswordMismatchError,
-)
-from controllers.console.error import AccountNotFound, EmailSendIpLimitError
+from controllers.console.auth.error import EmailCodeError, InvalidEmailError, InvalidTokenError, PasswordMismatchError
+from controllers.console.error import AccountInFreezeError, AccountNotFound, EmailSendIpLimitError
 from controllers.console.wraps import setup_required
 from events.tenant_event import tenant_was_created
 from extensions.ext_database import db
@@ -20,6 +17,7 @@ from libs.helper import email, extract_remote_ip
 from libs.password import hash_password, valid_password
 from models.account import Account
 from services.account_service import AccountService, TenantService
+from services.errors.account import AccountRegisterError
 from services.errors.workspace import WorkSpaceNotAllowedCreateError
 from services.feature_service import FeatureService
 
@@ -41,7 +39,8 @@ class ForgotPasswordSendEmailApi(Resource):
         else:
             language = "en-US"
 
-        account = Account.query.filter_by(email=args["email"]).first()
+        with Session(db.engine) as session:
+            account = session.execute(select(Account).filter_by(email=args["email"])).scalar_one_or_none()
         token = None
         if account is None:
             if FeatureService.get_system_features().is_allow_register:
@@ -108,7 +107,8 @@ class ForgotPasswordResetApi(Resource):
         password_hashed = hash_password(new_password, salt)
         base64_password_hashed = base64.b64encode(password_hashed).decode()
 
-        account = Account.query.filter_by(email=reset_data.get("email")).first()
+        with Session(db.engine) as session:
+            account = session.execute(select(Account).filter_by(email=reset_data.get("email"))).scalar_one_or_none()
         if account:
             account.password = base64_password_hashed
             account.password_salt = base64_salt
@@ -122,13 +122,15 @@ class ForgotPasswordResetApi(Resource):
         else:
             try:
                 account = AccountService.create_account_and_tenant(
-                    email=reset_data.get("email"),
-                    name=reset_data.get("email"),
+                    email=reset_data.get("email", ""),
+                    name=reset_data.get("email", ""),
                     password=password_confirm,
                     interface_language=languages[0],
                 )
             except WorkSpaceNotAllowedCreateError:
                 pass
+            except AccountRegisterError:
+                raise AccountInFreezeError()
 
         return {"result": "success"}
 

api/controllers/console/auth/login.py

@@ -1,10 +1,11 @@
 from typing import cast
 
-import flask_login
+import flask_login  # type: ignore
 from flask import request
-from flask_restful import Resource, reqparse
+from flask_restful import Resource, reqparse  # type: ignore
 
 import services
+from configs import dify_config
 from constants.languages import languages
 from controllers.console import api
 from controllers.console.auth.error import (
@@ -16,6 +17,7 @@ from controllers.console.auth.error import (
 )
 from controllers.console.error import (
     AccountBannedError,
+    AccountInFreezeError,
     AccountNotFound,
     EmailSendIpLimitError,
     NotAllowedCreateWorkspace,
@@ -26,6 +28,8 @@ from libs.helper import email, extract_remote_ip
 from libs.password import valid_password
 from models.account import Account
 from services.account_service import AccountService, RegisterService, TenantService
+from services.billing_service import BillingService
+from services.errors.account import AccountRegisterError
 from services.errors.workspace import WorkSpaceNotAllowedCreateError
 from services.feature_service import FeatureService
 
@@ -44,6 +48,9 @@ class LoginApi(Resource):
         parser.add_argument("language", type=str, required=False, default="en-US", location="json")
         args = parser.parse_args()
 
+        if dify_config.BILLING_ENABLED and BillingService.is_email_in_freeze(args["email"]):
+            raise AccountInFreezeError()
+
         is_login_error_rate_limit = AccountService.is_login_error_rate_limit(args["email"])
         if is_login_error_rate_limit:
             raise EmailPasswordLoginLimitError()
@@ -113,8 +120,10 @@ class ResetPasswordSendEmailApi(Resource):
             language = "zh-Hans"
         else:
             language = "en-US"
-
-        account = AccountService.get_user_through_email(args["email"])
+        try:
+            account = AccountService.get_user_through_email(args["email"])
+        except AccountRegisterError as are:
+            raise AccountInFreezeError()
         if account is None:
             if FeatureService.get_system_features().is_allow_register:
                 token = AccountService.send_reset_password_email(email=args["email"], language=language)
@@ -142,8 +151,11 @@ class EmailCodeLoginSendEmailApi(Resource):
             language = "zh-Hans"
         else:
             language = "en-US"
+        try:
+            account = AccountService.get_user_through_email(args["email"])
+        except AccountRegisterError as are:
+            raise AccountInFreezeError()
 
-        account = AccountService.get_user_through_email(args["email"])
         if account is None:
             if FeatureService.get_system_features().is_allow_register:
                 token = AccountService.send_email_code_login_email(email=args["email"], language=language)
@@ -177,7 +189,10 @@ class EmailCodeLoginApi(Resource):
             raise EmailCodeError()
 
         AccountService.revoke_email_code_login_token(args["token"])
-        account = AccountService.get_user_through_email(user_email)
+        try:
+            account = AccountService.get_user_through_email(user_email)
+        except AccountRegisterError as are:
+            raise AccountInFreezeError()
         if account:
             tenant = TenantService.get_join_tenants(account)
             if not tenant:
@@ -196,6 +211,8 @@ class EmailCodeLoginApi(Resource):
                 )
             except WorkSpaceNotAllowedCreateError:
                 return NotAllowedCreateWorkspace()
+            except AccountRegisterError as are:
+                raise AccountInFreezeError()
         token_pair = AccountService.login(account, ip_address=extract_remote_ip(request))
         AccountService.reset_login_error_rate_limit(args["email"])
         return {"result": "success", "data": token_pair.model_dump()}

api/controllers/console/auth/oauth.py

@@ -4,7 +4,9 @@ from typing import Optional
 
 import requests
 from flask import current_app, redirect, request
-from flask_restful import Resource
+from flask_restful import Resource  # type: ignore
+from sqlalchemy import select
+from sqlalchemy.orm import Session
 from werkzeug.exceptions import Unauthorized
 
 from configs import dify_config
@@ -16,7 +18,7 @@ from libs.oauth import GitHubOAuth, GoogleOAuth, OAuthUserInfo
 from models import Account
 from models.account import AccountStatus
 from services.account_service import AccountService, RegisterService, TenantService
-from services.errors.account import AccountNotFoundError
+from services.errors.account import AccountNotFoundError, AccountRegisterError
 from services.errors.workspace import WorkSpaceNotAllowedCreateError, WorkSpaceNotFoundError
 from services.feature_service import FeatureService
 
@@ -76,8 +78,9 @@ class OAuthCallback(Resource):
         try:
             token = oauth_provider.get_access_token(code)
             user_info = oauth_provider.get_user_info(token)
-        except requests.exceptions.HTTPError as e:
-            logging.exception(f"An error occurred during the OAuth process with {provider}: {e.response.text}")
+        except requests.exceptions.RequestException as e:
+            error_text = e.response.text if e.response else str(e)
+            logging.exception(f"An error occurred during the OAuth process with {provider}: {error_text}")
             return {"error": "OAuth process failed"}, 400
 
         if invite_token and RegisterService.is_valid_invite_token(invite_token):
@@ -98,6 +101,8 @@ class OAuthCallback(Resource):
                 f"{dify_config.CONSOLE_WEB_URL}/signin"
                 "?message=Workspace not found, please contact system admin to invite you to join in a workspace."
             )
+        except AccountRegisterError as e:
+            return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin?message={e.description}")
 
         # Check account status
         if account.status == AccountStatus.BANNED.value:
@@ -129,10 +134,11 @@ class OAuthCallback(Resource):
 
 
 def _get_account_by_openid_or_email(provider: str, user_info: OAuthUserInfo) -> Optional[Account]:
-    account = Account.get_by_openid(provider, user_info.id)
+    account: Optional[Account] = Account.get_by_openid(provider, user_info.id)
 
     if not account:
-        account = Account.query.filter_by(email=user_info.email).first()
+        with Session(db.engine) as session:
+            account = session.execute(select(Account).filter_by(email=user_info.email)).scalar_one_or_none()
 
     return account
 

api/controllers/console/billing/billing.py

@@ -1,5 +1,5 @@
-from flask_login import current_user
-from flask_restful import Resource, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, reqparse  # type: ignore
 
 from controllers.console import api
 from controllers.console.wraps import account_initialization_required, only_edition_cloud, setup_required

api/controllers/console/datasets/data_source.py

@@ -2,8 +2,10 @@ import datetime
 import json
 
 from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from sqlalchemy import select
+from sqlalchemy.orm import Session
 from werkzeug.exceptions import NotFound
 
 from controllers.console import api
@@ -76,7 +78,10 @@ class DataSourceApi(Resource):
     def patch(self, binding_id, action):
         binding_id = str(binding_id)
         action = str(action)
-        data_source_binding = DataSourceOauthBinding.query.filter_by(id=binding_id).first()
+        with Session(db.engine) as session:
+            data_source_binding = session.execute(
+                select(DataSourceOauthBinding).filter_by(id=binding_id)
+            ).scalar_one_or_none()
         if data_source_binding is None:
             raise NotFound("Data source binding not found.")
         # enable binding
@@ -108,47 +113,53 @@ class DataSourceNotionListApi(Resource):
     def get(self):
         dataset_id = request.args.get("dataset_id", default=None, type=str)
         exist_page_ids = []
-        # import notion in the exist dataset
-        if dataset_id:
-            dataset = DatasetService.get_dataset(dataset_id)
-            if not dataset:
-                raise NotFound("Dataset not found.")
-            if dataset.data_source_type != "notion_import":
-                raise ValueError("Dataset is not notion type.")
-            documents = Document.query.filter_by(
-                dataset_id=dataset_id,
-                tenant_id=current_user.current_tenant_id,
-                data_source_type="notion_import",
-                enabled=True,
+        with Session(db.engine) as session:
+            # import notion in the exist dataset
+            if dataset_id:
+                dataset = DatasetService.get_dataset(dataset_id)
+                if not dataset:
+                    raise NotFound("Dataset not found.")
+                if dataset.data_source_type != "notion_import":
+                    raise ValueError("Dataset is not notion type.")
+
+                documents = session.execute(
+                    select(Document).filter_by(
+                        dataset_id=dataset_id,
+                        tenant_id=current_user.current_tenant_id,
+                        data_source_type="notion_import",
+                        enabled=True,
+                    )
+                ).all()
+                if documents:
+                    for document in documents:
+                        data_source_info = json.loads(document.data_source_info)
+                        exist_page_ids.append(data_source_info["notion_page_id"])
+            # get all authorized pages
+            data_source_bindings = session.execute(
+                select(DataSourceOauthBinding).filter_by(
+                    tenant_id=current_user.current_tenant_id, provider="notion", disabled=False
+                )
             ).all()
-            if documents:
-                for document in documents:
-                    data_source_info = json.loads(document.data_source_info)
-                    exist_page_ids.append(data_source_info["notion_page_id"])
-        # get all authorized pages
-        data_source_bindings = DataSourceOauthBinding.query.filter_by(
-            tenant_id=current_user.current_tenant_id, provider="notion", disabled=False
-        ).all()
-        if not data_source_bindings:
-            return {"notion_info": []}, 200
-        pre_import_info_list = []
-        for data_source_binding in data_source_bindings:
-            source_info = data_source_binding.source_info
-            pages = source_info["pages"]
-            # Filter out already bound pages
-            for page in pages:
-                if page["page_id"] in exist_page_ids:
-                    page["is_bound"] = True
-                else:
-                    page["is_bound"] = False
-            pre_import_info = {
-                "workspace_name": source_info["workspace_name"],
-                "workspace_icon": source_info["workspace_icon"],
-                "workspace_id": source_info["workspace_id"],
-                "pages": pages,
-            }
-            pre_import_info_list.append(pre_import_info)
-        return {"notion_info": pre_import_info_list}, 200
+            if not data_source_bindings:
+                return {"notion_info": []}, 200
+            pre_import_info_list = []
+            for data_source_binding in data_source_bindings:
+                source_info = data_source_binding.source_info
+                pages = source_info["pages"]
+                # Filter out already bound pages
+                for page in pages:
+                    if page["page_id"] in exist_page_ids:
+                        page["is_bound"] = True
+                    else:
+                        page["is_bound"] = False
+                pre_import_info = {
+                    "workspace_name": source_info["workspace_name"],
+                    "workspace_icon": source_info["workspace_icon"],
+                    "workspace_id": source_info["workspace_id"],
+                    "pages": pages,
+                }
+                pre_import_info_list.append(pre_import_info)
+            return {"notion_info": pre_import_info_list}, 200
 
 
 class DataSourceNotionApi(Resource):
@@ -158,14 +169,17 @@ class DataSourceNotionApi(Resource):
     def get(self, workspace_id, page_id, page_type):
         workspace_id = str(workspace_id)
         page_id = str(page_id)
-        data_source_binding = DataSourceOauthBinding.query.filter(
-            db.and_(
-                DataSourceOauthBinding.tenant_id == current_user.current_tenant_id,
-                DataSourceOauthBinding.provider == "notion",
-                DataSourceOauthBinding.disabled == False,
-                DataSourceOauthBinding.source_info["workspace_id"] == f'"{workspace_id}"',
-            )
-        ).first()
+        with Session(db.engine) as session:
+            data_source_binding = session.execute(
+                select(DataSourceOauthBinding).filter(
+                    db.and_(
+                        DataSourceOauthBinding.tenant_id == current_user.current_tenant_id,
+                        DataSourceOauthBinding.provider == "notion",
+                        DataSourceOauthBinding.disabled == False,
+                        DataSourceOauthBinding.source_info["workspace_id"] == f'"{workspace_id}"',
+                    )
+                )
+            ).scalar_one_or_none()
         if not data_source_binding:
             raise NotFound("Data source binding not found.")
 
@@ -218,7 +232,7 @@ class DataSourceNotionApi(Resource):
             args["doc_form"],
             args["doc_language"],
         )
-        return response, 200
+        return response.model_dump(), 200
 
 
 class DataSourceNotionDatasetSyncApi(Resource):

api/controllers/console/datasets/datasets.py

@@ -1,7 +1,7 @@
-import flask_restful
+import flask_restful  # type: ignore
 from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal, marshal_with, reqparse
+from flask_login import current_user  # type: ignore  # type: ignore
+from flask_restful import Resource, marshal, marshal_with, reqparse  # type: ignore
 from werkzeug.exceptions import Forbidden, NotFound
 
 import services
@@ -464,7 +464,7 @@ class DatasetIndexingEstimateApi(Resource):
         except Exception as e:
             raise IndexingEstimateError(str(e))
 
-        return response, 200
+        return response.model_dump(), 200
 
 
 class DatasetRelatedAppListApi(Resource):
@@ -640,6 +640,7 @@ class DatasetRetrievalSettingApi(Resource):
                 | VectorType.MYSCALE
                 | VectorType.ORACLE
                 | VectorType.ELASTICSEARCH
+                | VectorType.ELASTICSEARCH_JA
                 | VectorType.PGVECTOR
                 | VectorType.TIDB_ON_QDRANT
                 | VectorType.LINDORM
@@ -683,6 +684,7 @@ class DatasetRetrievalSettingMockApi(Resource):
                 | VectorType.MYSCALE
                 | VectorType.ORACLE
                 | VectorType.ELASTICSEARCH
+                | VectorType.ELASTICSEARCH_JA
                 | VectorType.COUCHBASE
                 | VectorType.PGVECTOR
                 | VectorType.LINDORM
@@ -733,6 +735,18 @@ class DatasetPermissionUserListApi(Resource):
         }, 200
 
 
+class DatasetAutoDisableLogApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        return DatasetService.get_dataset_auto_disable_logs(dataset_id_str), 200
+
+
 api.add_resource(DatasetListApi, "/datasets")
 api.add_resource(DatasetApi, "/datasets/<uuid:dataset_id>")
 api.add_resource(DatasetUseCheckApi, "/datasets/<uuid:dataset_id>/use-check")
@@ -747,3 +761,4 @@ api.add_resource(DatasetApiBaseUrlApi, "/datasets/api-base-info")
 api.add_resource(DatasetRetrievalSettingApi, "/datasets/retrieval-setting")
 api.add_resource(DatasetRetrievalSettingMockApi, "/datasets/retrieval-setting/<string:vector_type>")
 api.add_resource(DatasetPermissionUserListApi, "/datasets/<uuid:dataset_id>/permission-part-users")
+api.add_resource(DatasetAutoDisableLogApi, "/datasets/<uuid:dataset_id>/auto-disable-logs")

api/controllers/console/datasets/datasets_document.py

@@ -1,12 +1,12 @@
 import logging
 from argparse import ArgumentTypeError
 from datetime import UTC, datetime
+from typing import cast
 
 from flask import request
-from flask_login import current_user
-from flask_restful import Resource, fields, marshal, marshal_with, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, fields, marshal, marshal_with, reqparse  # type: ignore
 from sqlalchemy import asc, desc
-from transformers.hf_argparser import string_to_bool
 from werkzeug.exceptions import Forbidden, NotFound
 
 import services
@@ -39,6 +39,7 @@ from core.indexing_runner import IndexingRunner
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.invoke import InvokeAuthorizationError
+from core.plugin.manager.exc import PluginDaemonClientSideError
 from core.rag.extractor.entity.extract_setting import ExtractSetting
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
@@ -51,6 +52,7 @@ from fields.document_fields import (
 from libs.login import login_required
 from models import Dataset, DatasetProcessRule, Document, DocumentSegment, UploadFile
 from services.dataset_service import DatasetService, DocumentService
+from services.entities.knowledge_entities.knowledge_entities import KnowledgeConfig
 from tasks.add_document_to_index_task import add_document_to_index_task
 from tasks.remove_document_from_index_task import remove_document_from_index_task
 
@@ -148,8 +150,20 @@ class DatasetDocumentListApi(Resource):
         sort = request.args.get("sort", default="-created_at", type=str)
         # "yes", "true", "t", "y", "1" convert to True, while others convert to False.
         try:
-            fetch = string_to_bool(request.args.get("fetch", default="false"))
-        except (ArgumentTypeError, ValueError, Exception) as e:
+            fetch_val = request.args.get("fetch", default="false")
+            if isinstance(fetch_val, bool):
+                fetch = fetch_val
+            else:
+                if fetch_val.lower() in ("yes", "true", "t", "y", "1"):
+                    fetch = True
+                elif fetch_val.lower() in ("no", "false", "f", "n", "0"):
+                    fetch = False
+                else:
+                    raise ArgumentTypeError(
+                        f"Truthy value expected: got {fetch_val} but expected one of yes/no, true/false, t/f, y/n, 1/0 "
+                        f"(case insensitive)."
+                    )
+        except (ArgumentTypeError, ValueError, Exception):
             fetch = False
         dataset = DatasetService.get_dataset(dataset_id)
         if not dataset:
@@ -254,20 +268,23 @@ class DatasetDocumentListApi(Resource):
         parser.add_argument("duplicate", type=bool, default=True, nullable=False, location="json")
         parser.add_argument("original_document_id", type=str, required=False, location="json")
         parser.add_argument("doc_form", type=str, default="text_model", required=False, nullable=False, location="json")
+        parser.add_argument("retrieval_model", type=dict, required=False, nullable=False, location="json")
+        parser.add_argument("embedding_model", type=str, required=False, nullable=True, location="json")
+        parser.add_argument("embedding_model_provider", type=str, required=False, nullable=True, location="json")
         parser.add_argument(
             "doc_language", type=str, default="English", required=False, nullable=False, location="json"
         )
-        parser.add_argument("retrieval_model", type=dict, required=False, nullable=False, location="json")
         args = parser.parse_args()
+        knowledge_config = KnowledgeConfig(**args)
 
-        if not dataset.indexing_technique and not args["indexing_technique"]:
+        if not dataset.indexing_technique and not knowledge_config.indexing_technique:
             raise ValueError("indexing_technique is required.")
 
         # validate args
-        DocumentService.document_create_args_validate(args)
+        DocumentService.document_create_args_validate(knowledge_config)
 
         try:
-            documents, batch = DocumentService.save_document_with_dataset_id(dataset, args, current_user)
+            documents, batch = DocumentService.save_document_with_dataset_id(dataset, knowledge_config, current_user)
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
         except QuotaExceededError:
@@ -277,6 +294,25 @@ class DatasetDocumentListApi(Resource):
 
         return {"documents": documents, "batch": batch}
 
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, dataset_id):
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+
+        try:
+            document_ids = request.args.getlist("document_id")
+            DocumentService.delete_documents(dataset, document_ids)
+        except services.errors.document.DocumentIndexingError:
+            raise DocumentIndexingError("Cannot delete document during indexing.")
+
+        return {"result": "success"}, 204
+
 
 class DatasetInitApi(Resource):
     @setup_required
@@ -312,9 +348,9 @@ class DatasetInitApi(Resource):
         # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
         if not current_user.is_dataset_editor:
             raise Forbidden()
-
-        if args["indexing_technique"] == "high_quality":
-            if args["embedding_model"] is None or args["embedding_model_provider"] is None:
+        knowledge_config = KnowledgeConfig(**args)
+        if knowledge_config.indexing_technique == "high_quality":
+            if knowledge_config.embedding_model is None or knowledge_config.embedding_model_provider is None:
                 raise ValueError("embedding model and embedding model provider are required for high quality indexing.")
             try:
                 model_manager = ModelManager()
@@ -333,11 +369,11 @@ class DatasetInitApi(Resource):
                 raise ProviderNotInitializeError(ex.description)
 
         # validate args
-        DocumentService.document_create_args_validate(args)
+        DocumentService.document_create_args_validate(knowledge_config)
 
         try:
             dataset, documents, batch = DocumentService.save_document_without_dataset_id(
-                tenant_id=current_user.current_tenant_id, document_data=args, account=current_user
+                tenant_id=current_user.current_tenant_id, knowledge_config=knowledge_config, account=current_user
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
@@ -390,7 +426,7 @@ class DocumentIndexingEstimateApi(DocumentResource):
                 indexing_runner = IndexingRunner()
 
                 try:
-                    response = indexing_runner.indexing_estimate(
+                    estimate_response = indexing_runner.indexing_estimate(
                         current_user.current_tenant_id,
                         [extract_setting],
                         data_process_rule_dict,
@@ -398,6 +434,7 @@ class DocumentIndexingEstimateApi(DocumentResource):
                         "English",
                         dataset_id,
                     )
+                    return estimate_response.model_dump(), 200
                 except LLMBadRequestError:
                     raise ProviderNotInitializeError(
                         "No Embedding Model available. Please configure a valid provider "
@@ -405,10 +442,12 @@ class DocumentIndexingEstimateApi(DocumentResource):
                     )
                 except ProviderTokenNotInitError as ex:
                     raise ProviderNotInitializeError(ex.description)
+                except PluginDaemonClientSideError as ex:
+                    raise ProviderNotInitializeError(ex.description)
                 except Exception as e:
                     raise IndexingEstimateError(str(e))
 
-        return response
+        return response, 200
 
 
 class DocumentBatchIndexingEstimateApi(DocumentResource):
@@ -419,9 +458,8 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
         dataset_id = str(dataset_id)
         batch = str(batch)
         documents = self.get_batch_documents(dataset_id, batch)
-        response = {"tokens": 0, "total_price": 0, "currency": "USD", "total_segments": 0, "preview": []}
         if not documents:
-            return response
+            return {"tokens": 0, "total_price": 0, "currency": "USD", "total_segments": 0, "preview": []}, 200
         data_process_rule = documents[0].dataset_process_rule
         data_process_rule_dict = data_process_rule.to_dict()
         info_list = []
@@ -499,6 +537,7 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
                     "English",
                     dataset_id,
                 )
+                return response.model_dump(), 200
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
                     "No Embedding Model available. Please configure a valid provider "
@@ -506,9 +545,10 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
+            except PluginDaemonClientSideError as ex:
+                raise ProviderNotInitializeError(ex.description)
             except Exception as e:
                 raise IndexingEstimateError(str(e))
-        return response
 
 
 class DocumentBatchIndexingStatusApi(DocumentResource):
@@ -581,7 +621,8 @@ class DocumentDetailApi(DocumentResource):
         if metadata == "only":
             response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata}
         elif metadata == "without":
-            process_rules = DatasetService.get_process_rules(dataset_id)
+            dataset_process_rules = DatasetService.get_process_rules(dataset_id)
+            document_process_rules = document.dataset_process_rule.to_dict()
             data_source_info = document.data_source_detail_dict
             response = {
                 "id": document.id,
@@ -589,7 +630,8 @@ class DocumentDetailApi(DocumentResource):
                 "data_source_type": document.data_source_type,
                 "data_source_info": data_source_info,
                 "dataset_process_rule_id": document.dataset_process_rule_id,
-                "dataset_process_rule": process_rules,
+                "dataset_process_rule": dataset_process_rules,
+                "document_process_rule": document_process_rules,
                 "name": document.name,
                 "created_from": document.created_from,
                 "created_by": document.created_by,
@@ -612,7 +654,8 @@ class DocumentDetailApi(DocumentResource):
                 "doc_language": document.doc_language,
             }
         else:
-            process_rules = DatasetService.get_process_rules(dataset_id)
+            dataset_process_rules = DatasetService.get_process_rules(dataset_id)
+            document_process_rules = document.dataset_process_rule.to_dict()
             data_source_info = document.data_source_detail_dict
             response = {
                 "id": document.id,
@@ -620,7 +663,8 @@ class DocumentDetailApi(DocumentResource):
                 "data_source_type": document.data_source_type,
                 "data_source_info": data_source_info,
                 "dataset_process_rule_id": document.dataset_process_rule_id,
-                "dataset_process_rule": process_rules,
+                "dataset_process_rule": dataset_process_rules,
+                "document_process_rule": document_process_rules,
                 "name": document.name,
                 "created_from": document.created_from,
                 "created_by": document.created_by,
@@ -733,8 +777,7 @@ class DocumentMetadataApi(DocumentResource):
 
         if not isinstance(doc_metadata, dict):
             raise ValueError("doc_metadata must be a dictionary.")
-
-        metadata_schema = DocumentService.DOCUMENT_METADATA_SCHEMA[doc_type]
+        metadata_schema: dict = cast(dict, DocumentService.DOCUMENT_METADATA_SCHEMA[doc_type])
 
         document.doc_metadata = {}
         if doc_type == "others":
@@ -757,9 +800,8 @@ class DocumentStatusApi(DocumentResource):
     @login_required
     @account_initialization_required
     @cloud_edition_billing_resource_check("vector_space")
-    def patch(self, dataset_id, document_id, action):
+    def patch(self, dataset_id, action):
         dataset_id = str(dataset_id)
-        document_id = str(document_id)
         dataset = DatasetService.get_dataset(dataset_id)
         if dataset is None:
             raise NotFound("Dataset not found.")
@@ -774,84 +816,79 @@ class DocumentStatusApi(DocumentResource):
         # check user's permission
         DatasetService.check_dataset_permission(dataset, current_user)
 
-        document = self.get_document(dataset_id, document_id)
+        document_ids = request.args.getlist("document_id")
+        for document_id in document_ids:
+            document = self.get_document(dataset_id, document_id)
 
-        indexing_cache_key = "document_{}_indexing".format(document.id)
-        cache_result = redis_client.get(indexing_cache_key)
-        if cache_result is not None:
-            raise InvalidActionError("Document is being indexed, please try again later")
+            indexing_cache_key = "document_{}_indexing".format(document.id)
+            cache_result = redis_client.get(indexing_cache_key)
+            if cache_result is not None:
+                raise InvalidActionError(f"Document:{document.name} is being indexed, please try again later")
 
-        if action == "enable":
-            if document.enabled:
-                raise InvalidActionError("Document already enabled.")
+            if action == "enable":
+                if document.enabled:
+                    continue
+                document.enabled = True
+                document.disabled_at = None
+                document.disabled_by = None
+                document.updated_at = datetime.now(UTC).replace(tzinfo=None)
+                db.session.commit()
 
-            document.enabled = True
-            document.disabled_at = None
-            document.disabled_by = None
-            document.updated_at = datetime.now(UTC).replace(tzinfo=None)
-            db.session.commit()
+                # Set cache to prevent indexing the same document multiple times
+                redis_client.setex(indexing_cache_key, 600, 1)
 
-            # Set cache to prevent indexing the same document multiple times
-            redis_client.setex(indexing_cache_key, 600, 1)
+                add_document_to_index_task.delay(document_id)
 
-            add_document_to_index_task.delay(document_id)
+            elif action == "disable":
+                if not document.completed_at or document.indexing_status != "completed":
+                    raise InvalidActionError(f"Document: {document.name} is not completed.")
+                if not document.enabled:
+                    continue
 
-            return {"result": "success"}, 200
+                document.enabled = False
+                document.disabled_at = datetime.now(UTC).replace(tzinfo=None)
+                document.disabled_by = current_user.id
+                document.updated_at = datetime.now(UTC).replace(tzinfo=None)
+                db.session.commit()
 
-        elif action == "disable":
-            if not document.completed_at or document.indexing_status != "completed":
-                raise InvalidActionError("Document is not completed.")
-            if not document.enabled:
-                raise InvalidActionError("Document already disabled.")
-
-            document.enabled = False
-            document.disabled_at = datetime.now(UTC).replace(tzinfo=None)
-            document.disabled_by = current_user.id
-            document.updated_at = datetime.now(UTC).replace(tzinfo=None)
-            db.session.commit()
-
-            # Set cache to prevent indexing the same document multiple times
-            redis_client.setex(indexing_cache_key, 600, 1)
-
-            remove_document_from_index_task.delay(document_id)
-
-            return {"result": "success"}, 200
-
-        elif action == "archive":
-            if document.archived:
-                raise InvalidActionError("Document already archived.")
-
-            document.archived = True
-            document.archived_at = datetime.now(UTC).replace(tzinfo=None)
-            document.archived_by = current_user.id
-            document.updated_at = datetime.now(UTC).replace(tzinfo=None)
-            db.session.commit()
-
-            if document.enabled:
                 # Set cache to prevent indexing the same document multiple times
                 redis_client.setex(indexing_cache_key, 600, 1)
 
                 remove_document_from_index_task.delay(document_id)
 
-            return {"result": "success"}, 200
-        elif action == "un_archive":
-            if not document.archived:
-                raise InvalidActionError("Document is not archived.")
+            elif action == "archive":
+                if document.archived:
+                    continue
 
-            document.archived = False
-            document.archived_at = None
-            document.archived_by = None
-            document.updated_at = datetime.now(UTC).replace(tzinfo=None)
-            db.session.commit()
+                document.archived = True
+                document.archived_at = datetime.now(UTC).replace(tzinfo=None)
+                document.archived_by = current_user.id
+                document.updated_at = datetime.now(UTC).replace(tzinfo=None)
+                db.session.commit()
 
-            # Set cache to prevent indexing the same document multiple times
-            redis_client.setex(indexing_cache_key, 600, 1)
+                if document.enabled:
+                    # Set cache to prevent indexing the same document multiple times
+                    redis_client.setex(indexing_cache_key, 600, 1)
 
-            add_document_to_index_task.delay(document_id)
+                    remove_document_from_index_task.delay(document_id)
 
-            return {"result": "success"}, 200
-        else:
-            raise InvalidActionError()
+            elif action == "un_archive":
+                if not document.archived:
+                    continue
+                document.archived = False
+                document.archived_at = None
+                document.archived_by = None
+                document.updated_at = datetime.now(UTC).replace(tzinfo=None)
+                db.session.commit()
+
+                # Set cache to prevent indexing the same document multiple times
+                redis_client.setex(indexing_cache_key, 600, 1)
+
+                add_document_to_index_task.delay(document_id)
+
+            else:
+                raise InvalidActionError()
+        return {"result": "success"}, 200
 
 
 class DocumentPauseApi(DocumentResource):
@@ -948,7 +985,7 @@ class DocumentRetryApi(DocumentResource):
                 if document.indexing_status == "completed":
                     raise DocumentAlreadyFinishedError()
                 retry_documents.append(document)
-            except Exception as e:
+            except Exception:
                 logging.exception(f"Failed to retry document, document id: {document_id}")
                 continue
         # retry document
@@ -1022,7 +1059,7 @@ api.add_resource(
 )
 api.add_resource(DocumentDeleteApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
 api.add_resource(DocumentMetadataApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/metadata")
-api.add_resource(DocumentStatusApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/status/<string:action>")
+api.add_resource(DocumentStatusApi, "/datasets/<uuid:dataset_id>/documents/status/<string:action>/batch")
 api.add_resource(DocumentPauseApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/processing/pause")
 api.add_resource(DocumentRecoverApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/processing/resume")
 api.add_resource(DocumentRetryApi, "/datasets/<uuid:dataset_id>/retry")

api/controllers/console/datasets/datasets_segments.py

@@ -1,16 +1,21 @@
 import uuid
-from datetime import UTC, datetime
 
 import pandas as pd
 from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, marshal, reqparse  # type: ignore
 from werkzeug.exceptions import Forbidden, NotFound
 
 import services
 from controllers.console import api
 from controllers.console.app.error import ProviderNotInitializeError
-from controllers.console.datasets.error import InvalidActionError, NoFileUploadedError, TooManyFilesError
+from controllers.console.datasets.error import (
+    ChildChunkDeleteIndexError,
+    ChildChunkIndexingError,
+    InvalidActionError,
+    NoFileUploadedError,
+    TooManyFilesError,
+)
 from controllers.console.wraps import (
     account_initialization_required,
     cloud_edition_billing_knowledge_limit_check,
@@ -20,15 +25,15 @@ from controllers.console.wraps import (
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
-from extensions.ext_database import db
 from extensions.ext_redis import redis_client
-from fields.segment_fields import segment_fields
+from fields.segment_fields import child_chunk_fields, segment_fields
 from libs.login import login_required
-from models import DocumentSegment
+from models.dataset import ChildChunk, DocumentSegment
 from services.dataset_service import DatasetService, DocumentService, SegmentService
+from services.entities.knowledge_entities.knowledge_entities import ChildChunkUpdateArgs, SegmentUpdateArgs
+from services.errors.chunk import ChildChunkDeleteIndexError as ChildChunkDeleteIndexServiceError
+from services.errors.chunk import ChildChunkIndexingError as ChildChunkIndexingServiceError
 from tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task
-from tasks.disable_segment_from_index_task import disable_segment_from_index_task
-from tasks.enable_segment_to_index_task import enable_segment_to_index_task
 
 
 class DatasetDocumentSegmentListApi(Resource):
@@ -53,15 +58,16 @@ class DatasetDocumentSegmentListApi(Resource):
             raise NotFound("Document not found.")
 
         parser = reqparse.RequestParser()
-        parser.add_argument("last_id", type=str, default=None, location="args")
         parser.add_argument("limit", type=int, default=20, location="args")
         parser.add_argument("status", type=str, action="append", default=[], location="args")
         parser.add_argument("hit_count_gte", type=int, default=None, location="args")
         parser.add_argument("enabled", type=str, default="all", location="args")
         parser.add_argument("keyword", type=str, default=None, location="args")
+        parser.add_argument("page", type=int, default=1, location="args")
+
         args = parser.parse_args()
 
-        last_id = args["last_id"]
+        page = args["page"]
         limit = min(args["limit"], 100)
         status_list = args["status"]
         hit_count_gte = args["hit_count_gte"]
@@ -69,14 +75,7 @@ class DatasetDocumentSegmentListApi(Resource):
 
         query = DocumentSegment.query.filter(
             DocumentSegment.document_id == str(document_id), DocumentSegment.tenant_id == current_user.current_tenant_id
-        )
-
-        if last_id is not None:
-            last_segment = db.session.get(DocumentSegment, str(last_id))
-            if last_segment:
-                query = query.filter(DocumentSegment.position > last_segment.position)
-            else:
-                return {"data": [], "has_more": False, "limit": limit}, 200
+        ).order_by(DocumentSegment.position.asc())
 
         if status_list:
             query = query.filter(DocumentSegment.status.in_(status_list))
@@ -93,21 +92,44 @@ class DatasetDocumentSegmentListApi(Resource):
             elif args["enabled"].lower() == "false":
                 query = query.filter(DocumentSegment.enabled == False)
 
-        total = query.count()
-        segments = query.order_by(DocumentSegment.position).limit(limit + 1).all()
+        segments = query.paginate(page=page, per_page=limit, max_per_page=100, error_out=False)
 
-        has_more = False
-        if len(segments) > limit:
-            has_more = True
-            segments = segments[:-1]
-
-        return {
-            "data": marshal(segments, segment_fields),
-            "doc_form": document.doc_form,
-            "has_more": has_more,
+        response = {
+            "data": marshal(segments.items, segment_fields),
             "limit": limit,
-            "total": total,
-        }, 200
+            "total": segments.total,
+            "total_pages": segments.pages,
+            "page": page,
+        }
+        return response, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, dataset_id, document_id):
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        segment_ids = request.args.getlist("segment_id")
+
+        # The role of the current user in the ta table must be admin or owner
+        if not current_user.is_editor:
+            raise Forbidden()
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+        SegmentService.delete_segments(segment_ids, document, dataset)
+        return {"result": "success"}, 200
 
 
 class DatasetDocumentSegmentApi(Resource):
@@ -115,11 +137,15 @@ class DatasetDocumentSegmentApi(Resource):
     @login_required
     @account_initialization_required
     @cloud_edition_billing_resource_check("vector_space")
-    def patch(self, dataset_id, segment_id, action):
+    def patch(self, dataset_id, document_id, action):
         dataset_id = str(dataset_id)
         dataset = DatasetService.get_dataset(dataset_id)
         if not dataset:
             raise NotFound("Dataset not found.")
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
         # check user's model setting
         DatasetService.check_dataset_model_setting(dataset)
         # The role of the current user in the ta table must be admin, owner, or editor
@@ -147,59 +173,17 @@ class DatasetDocumentSegmentApi(Resource):
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
+        segment_ids = request.args.getlist("segment_id")
 
-        segment = DocumentSegment.query.filter(
-            DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id
-        ).first()
-
-        if not segment:
-            raise NotFound("Segment not found.")
-
-        if segment.status != "completed":
-            raise NotFound("Segment is not completed, enable or disable function is not allowed")
-
-        document_indexing_cache_key = "document_{}_indexing".format(segment.document_id)
+        document_indexing_cache_key = "document_{}_indexing".format(document.id)
         cache_result = redis_client.get(document_indexing_cache_key)
         if cache_result is not None:
             raise InvalidActionError("Document is being indexed, please try again later")
-
-        indexing_cache_key = "segment_{}_indexing".format(segment.id)
-        cache_result = redis_client.get(indexing_cache_key)
-        if cache_result is not None:
-            raise InvalidActionError("Segment is being indexed, please try again later")
-
-        if action == "enable":
-            if segment.enabled:
-                raise InvalidActionError("Segment is already enabled.")
-
-            segment.enabled = True
-            segment.disabled_at = None
-            segment.disabled_by = None
-            db.session.commit()
-
-            # Set cache to prevent indexing the same segment multiple times
-            redis_client.setex(indexing_cache_key, 600, 1)
-
-            enable_segment_to_index_task.delay(segment.id)
-
-            return {"result": "success"}, 200
-        elif action == "disable":
-            if not segment.enabled:
-                raise InvalidActionError("Segment is already disabled.")
-
-            segment.enabled = False
-            segment.disabled_at = datetime.now(UTC).replace(tzinfo=None)
-            segment.disabled_by = current_user.id
-            db.session.commit()
-
-            # Set cache to prevent indexing the same segment multiple times
-            redis_client.setex(indexing_cache_key, 600, 1)
-
-            disable_segment_from_index_task.delay(segment.id)
-
-            return {"result": "success"}, 200
-        else:
-            raise InvalidActionError()
+        try:
+            SegmentService.update_segments_status(segment_ids, action, dataset, document)
+        except Exception as e:
+            raise InvalidActionError(str(e))
+        return {"result": "success"}, 200
 
 
 class DatasetDocumentSegmentAddApi(Resource):
@@ -307,9 +291,12 @@ class DatasetDocumentSegmentUpdateApi(Resource):
         parser.add_argument("content", type=str, required=True, nullable=False, location="json")
         parser.add_argument("answer", type=str, required=False, nullable=True, location="json")
         parser.add_argument("keywords", type=list, required=False, nullable=True, location="json")
+        parser.add_argument(
+            "regenerate_child_chunks", type=bool, required=False, nullable=True, default=False, location="json"
+        )
         args = parser.parse_args()
         SegmentService.segment_create_args_validate(args, document)
-        segment = SegmentService.update_segment(args, segment, document, dataset)
+        segment = SegmentService.update_segment(SegmentUpdateArgs(**args), segment, document, dataset)
         return {"data": marshal(segment, segment_fields), "doc_form": document.doc_form}, 200
 
     @setup_required
@@ -412,8 +399,248 @@ class DatasetDocumentSegmentBatchImportApi(Resource):
         return {"job_id": job_id, "job_status": cache_result.decode()}, 200
 
 
+class ChildChunkAddApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_resource_check("vector_space")
+    @cloud_edition_billing_knowledge_limit_check("add_segment")
+    def post(self, dataset_id, document_id, segment_id):
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        # check segment
+        segment_id = str(segment_id)
+        segment = DocumentSegment.query.filter(
+            DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id
+        ).first()
+        if not segment:
+            raise NotFound("Segment not found.")
+        if not current_user.is_editor:
+            raise Forbidden()
+        # check embedding model setting
+        if dataset.indexing_technique == "high_quality":
+            try:
+                model_manager = ModelManager()
+                model_manager.get_model_instance(
+                    tenant_id=current_user.current_tenant_id,
+                    provider=dataset.embedding_model_provider,
+                    model_type=ModelType.TEXT_EMBEDDING,
+                    model=dataset.embedding_model,
+                )
+            except LLMBadRequestError:
+                raise ProviderNotInitializeError(
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
+                )
+            except ProviderTokenNotInitError as ex:
+                raise ProviderNotInitializeError(ex.description)
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+        # validate args
+        parser = reqparse.RequestParser()
+        parser.add_argument("content", type=str, required=True, nullable=False, location="json")
+        args = parser.parse_args()
+        try:
+            child_chunk = SegmentService.create_child_chunk(args.get("content"), segment, document, dataset)
+        except ChildChunkIndexingServiceError as e:
+            raise ChildChunkIndexingError(str(e))
+        return {"data": marshal(child_chunk, child_chunk_fields)}, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id, document_id, segment_id):
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        # check segment
+        segment_id = str(segment_id)
+        segment = DocumentSegment.query.filter(
+            DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id
+        ).first()
+        if not segment:
+            raise NotFound("Segment not found.")
+        parser = reqparse.RequestParser()
+        parser.add_argument("limit", type=int, default=20, location="args")
+        parser.add_argument("keyword", type=str, default=None, location="args")
+        parser.add_argument("page", type=int, default=1, location="args")
+
+        args = parser.parse_args()
+
+        page = args["page"]
+        limit = min(args["limit"], 100)
+        keyword = args["keyword"]
+
+        child_chunks = SegmentService.get_child_chunks(segment_id, document_id, dataset_id, page, limit, keyword)
+        return {
+            "data": marshal(child_chunks.items, child_chunk_fields),
+            "total": child_chunks.total,
+            "total_pages": child_chunks.pages,
+            "page": page,
+            "limit": limit,
+        }, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_resource_check("vector_space")
+    def patch(self, dataset_id, document_id, segment_id):
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+            # check segment
+        segment_id = str(segment_id)
+        segment = DocumentSegment.query.filter(
+            DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id
+        ).first()
+        if not segment:
+            raise NotFound("Segment not found.")
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+        # validate args
+        parser = reqparse.RequestParser()
+        parser.add_argument("chunks", type=list, required=True, nullable=False, location="json")
+        args = parser.parse_args()
+        try:
+            chunks = [ChildChunkUpdateArgs(**chunk) for chunk in args.get("chunks")]
+            child_chunks = SegmentService.update_child_chunks(chunks, segment, document, dataset)
+        except ChildChunkIndexingServiceError as e:
+            raise ChildChunkIndexingError(str(e))
+        return {"data": marshal(child_chunks, child_chunk_fields)}, 200
+
+
+class ChildChunkUpdateApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, dataset_id, document_id, segment_id, child_chunk_id):
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        # check segment
+        segment_id = str(segment_id)
+        segment = DocumentSegment.query.filter(
+            DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id
+        ).first()
+        if not segment:
+            raise NotFound("Segment not found.")
+        # check child chunk
+        child_chunk_id = str(child_chunk_id)
+        child_chunk = ChildChunk.query.filter(
+            ChildChunk.id == str(child_chunk_id), ChildChunk.tenant_id == current_user.current_tenant_id
+        ).first()
+        if not child_chunk:
+            raise NotFound("Child chunk not found.")
+        # The role of the current user in the ta table must be admin or owner
+        if not current_user.is_editor:
+            raise Forbidden()
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+        try:
+            SegmentService.delete_child_chunk(child_chunk, dataset)
+        except ChildChunkDeleteIndexServiceError as e:
+            raise ChildChunkDeleteIndexError(str(e))
+        return {"result": "success"}, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_resource_check("vector_space")
+    def patch(self, dataset_id, document_id, segment_id, child_chunk_id):
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+            # check segment
+        segment_id = str(segment_id)
+        segment = DocumentSegment.query.filter(
+            DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id
+        ).first()
+        if not segment:
+            raise NotFound("Segment not found.")
+        # check child chunk
+        child_chunk_id = str(child_chunk_id)
+        child_chunk = ChildChunk.query.filter(
+            ChildChunk.id == str(child_chunk_id), ChildChunk.tenant_id == current_user.current_tenant_id
+        ).first()
+        if not child_chunk:
+            raise NotFound("Child chunk not found.")
+        # The role of the current user in the ta table must be admin or owner
+        if not current_user.is_editor:
+            raise Forbidden()
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+        # validate args
+        parser = reqparse.RequestParser()
+        parser.add_argument("content", type=str, required=True, nullable=False, location="json")
+        args = parser.parse_args()
+        try:
+            child_chunk = SegmentService.update_child_chunk(
+                args.get("content"), child_chunk, segment, document, dataset
+            )
+        except ChildChunkIndexingServiceError as e:
+            raise ChildChunkIndexingError(str(e))
+        return {"data": marshal(child_chunk, child_chunk_fields)}, 200
+
+
 api.add_resource(DatasetDocumentSegmentListApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments")
-api.add_resource(DatasetDocumentSegmentApi, "/datasets/<uuid:dataset_id>/segments/<uuid:segment_id>/<string:action>")
+api.add_resource(
+    DatasetDocumentSegmentApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segment/<string:action>"
+)
 api.add_resource(DatasetDocumentSegmentAddApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segment")
 api.add_resource(
     DatasetDocumentSegmentUpdateApi,
@@ -424,3 +651,11 @@ api.add_resource(
     "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/batch_import",
     "/datasets/batch_import_status/<uuid:job_id>",
 )
+api.add_resource(
+    ChildChunkAddApi,
+    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>/child_chunks",
+)
+api.add_resource(
+    ChildChunkUpdateApi,
+    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>/child_chunks/<uuid:child_chunk_id>",
+)

api/controllers/console/datasets/error.py

@@ -89,3 +89,15 @@ class IndexingEstimateError(BaseHTTPException):
     error_code = "indexing_estimate_error"
     description = "Knowledge indexing estimate failed: {message}"
     code = 500
+
+
+class ChildChunkIndexingError(BaseHTTPException):
+    error_code = "child_chunk_indexing_error"
+    description = "Create child chunk index failed: {message}"
+    code = 500
+
+
+class ChildChunkDeleteIndexError(BaseHTTPException):
+    error_code = "child_chunk_delete_index_error"
+    description = "Delete child chunk index failed: {message}"
+    code = 500

api/controllers/console/datasets/external.py

@@ -1,6 +1,6 @@
 from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, marshal, reqparse  # type: ignore
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 import services

api/controllers/console/datasets/hit_testing.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource
+from flask_restful import Resource  # type: ignore
 
 from controllers.console import api
 from controllers.console.datasets.hit_testing_base import DatasetsHitTestingBase

api/controllers/console/datasets/hit_testing_base.py

@@ -1,7 +1,7 @@
 import logging
 
-from flask_login import current_user
-from flask_restful import marshal, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import marshal, reqparse  # type: ignore
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 import services.dataset_service

api/controllers/console/datasets/website.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource, reqparse
+from flask_restful import Resource, reqparse  # type: ignore
 
 from controllers.console import api
 from controllers.console.datasets.error import WebsiteCrawlError

api/controllers/console/error.py

@@ -92,3 +92,12 @@ class UnauthorizedAndForceLogout(BaseHTTPException):
     error_code = "unauthorized_and_force_logout"
     description = "Unauthorized and force logout."
     code = 401
+
+
+class AccountInFreezeError(BaseHTTPException):
+    error_code = "account_in_freeze"
+    code = 400
+    description = (
+        "This email account has been deleted within the past 30 days"
+        "and is temporarily unavailable for new account registration."
+    )

api/controllers/console/explore/audio.py

@@ -4,7 +4,6 @@ from flask import request
 from werkzeug.exceptions import InternalServerError
 
 import services
-from controllers.console import api
 from controllers.console.app.error import (
     AppUnavailableError,
     AudioTooLargeError,
@@ -67,7 +66,7 @@ class ChatAudioApi(InstalledAppResource):
 
 class ChatTextApi(InstalledAppResource):
     def post(self, installed_app):
-        from flask_restful import reqparse
+        from flask_restful import reqparse  # type: ignore
 
         app_model = installed_app.app
         try:
@@ -118,9 +117,3 @@ class ChatTextApi(InstalledAppResource):
         except Exception as e:
             logging.exception("internal server error.")
             raise InternalServerError()
-
-
-api.add_resource(ChatAudioApi, "/installed-apps/<uuid:installed_app_id>/audio-to-text", endpoint="installed_app_audio")
-api.add_resource(ChatTextApi, "/installed-apps/<uuid:installed_app_id>/text-to-audio", endpoint="installed_app_text")
-# api.add_resource(ChatTextApiWithMessageId, '/installed-apps/<uuid:installed_app_id>/text-to-audio/message-id',
-#                  endpoint='installed_app_text_with_message_id')

api/controllers/console/explore/completion.py

@@ -1,12 +1,11 @@
 import logging
 from datetime import UTC, datetime
 
-from flask_login import current_user
-from flask_restful import reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import reqparse  # type: ignore
 from werkzeug.exceptions import InternalServerError, NotFound
 
 import services
-from controllers.console import api
 from controllers.console.app.error import (
     AppUnavailableError,
     CompletionRequestError,
@@ -19,7 +18,11 @@ from controllers.console.explore.error import NotChatAppError, NotCompletionAppE
 from controllers.console.explore.wraps import InstalledAppResource
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom
-from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
+from core.errors.error import (
+    ModelCurrentlyNotSupportError,
+    ProviderTokenNotInitError,
+    QuotaExceededError,
+)
 from core.model_runtime.errors.invoke import InvokeError
 from extensions.ext_database import db
 from libs import helper
@@ -147,21 +150,3 @@ class ChatStopApi(InstalledAppResource):
         AppQueueManager.set_stop_flag(task_id, InvokeFrom.EXPLORE, current_user.id)
 
         return {"result": "success"}, 200
-
-
-api.add_resource(
-    CompletionApi, "/installed-apps/<uuid:installed_app_id>/completion-messages", endpoint="installed_app_completion"
-)
-api.add_resource(
-    CompletionStopApi,
-    "/installed-apps/<uuid:installed_app_id>/completion-messages/<string:task_id>/stop",
-    endpoint="installed_app_stop_completion",
-)
-api.add_resource(
-    ChatApi, "/installed-apps/<uuid:installed_app_id>/chat-messages", endpoint="installed_app_chat_completion"
-)
-api.add_resource(
-    ChatStopApi,
-    "/installed-apps/<uuid:installed_app_id>/chat-messages/<string:task_id>/stop",
-    endpoint="installed_app_stop_chat_completion",
-)

api/controllers/console/explore/conversation.py

@@ -1,12 +1,13 @@
-from flask_login import current_user
-from flask_restful import marshal_with, reqparse
-from flask_restful.inputs import int_range
+from flask_login import current_user  # type: ignore
+from flask_restful import marshal_with, reqparse  # type: ignore
+from flask_restful.inputs import int_range  # type: ignore
+from sqlalchemy.orm import Session
 from werkzeug.exceptions import NotFound
 
-from controllers.console import api
 from controllers.console.explore.error import NotChatAppError
 from controllers.console.explore.wraps import InstalledAppResource
 from core.app.entities.app_invoke_entities import InvokeFrom
+from extensions.ext_database import db
 from fields.conversation_fields import conversation_infinite_scroll_pagination_fields, simple_conversation_fields
 from libs.helper import uuid_value
 from models.model import AppMode
@@ -34,14 +35,16 @@ class ConversationListApi(InstalledAppResource):
             pinned = True if args["pinned"] == "true" else False
 
         try:
-            return WebConversationService.pagination_by_last_id(
-                app_model=app_model,
-                user=current_user,
-                last_id=args["last_id"],
-                limit=args["limit"],
-                invoke_from=InvokeFrom.EXPLORE,
-                pinned=pinned,
-            )
+            with Session(db.engine) as session:
+                return WebConversationService.pagination_by_last_id(
+                    session=session,
+                    app_model=app_model,
+                    user=current_user,
+                    last_id=args["last_id"],
+                    limit=args["limit"],
+                    invoke_from=InvokeFrom.EXPLORE,
+                    pinned=pinned,
+                )
         except LastConversationNotExistsError:
             raise NotFound("Last Conversation Not Exists.")
 
@@ -114,28 +117,3 @@ class ConversationUnPinApi(InstalledAppResource):
         WebConversationService.unpin(app_model, conversation_id, current_user)
 
         return {"result": "success"}
-
-
-api.add_resource(
-    ConversationRenameApi,
-    "/installed-apps/<uuid:installed_app_id>/conversations/<uuid:c_id>/name",
-    endpoint="installed_app_conversation_rename",
-)
-api.add_resource(
-    ConversationListApi, "/installed-apps/<uuid:installed_app_id>/conversations", endpoint="installed_app_conversations"
-)
-api.add_resource(
-    ConversationApi,
-    "/installed-apps/<uuid:installed_app_id>/conversations/<uuid:c_id>",
-    endpoint="installed_app_conversation",
-)
-api.add_resource(
-    ConversationPinApi,
-    "/installed-apps/<uuid:installed_app_id>/conversations/<uuid:c_id>/pin",
-    endpoint="installed_app_conversation_pin",
-)
-api.add_resource(
-    ConversationUnPinApi,
-    "/installed-apps/<uuid:installed_app_id>/conversations/<uuid:c_id>/unpin",
-    endpoint="installed_app_conversation_unpin",
-)

api/controllers/console/explore/installed_app.py

@@ -1,8 +1,9 @@
 from datetime import UTC, datetime
+from typing import Any
 
 from flask import request
-from flask_login import current_user
-from flask_restful import Resource, inputs, marshal_with, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, inputs, marshal_with, reqparse  # type: ignore
 from sqlalchemy import and_
 from werkzeug.exceptions import BadRequest, Forbidden, NotFound
 
@@ -34,7 +35,7 @@ class InstalledAppsListApi(Resource):
             installed_apps = db.session.query(InstalledApp).filter(InstalledApp.tenant_id == current_tenant_id).all()
 
         current_user.role = TenantService.get_user_role(current_user, current_user.current_tenant)
-        installed_apps = [
+        installed_app_list: list[dict[str, Any]] = [
             {
                 "id": installed_app.id,
                 "app": installed_app.app,
@@ -47,7 +48,7 @@ class InstalledAppsListApi(Resource):
             for installed_app in installed_apps
             if installed_app.app is not None
         ]
-        installed_apps.sort(
+        installed_app_list.sort(
             key=lambda app: (
                 -app["is_pinned"],
                 app["last_used_at"] is None,
@@ -55,7 +56,7 @@ class InstalledAppsListApi(Resource):
             )
         )
 
-        return {"installed_apps": installed_apps}
+        return {"installed_apps": installed_app_list}
 
     @login_required
     @account_initialization_required

api/controllers/console/explore/message.py

@@ -1,12 +1,11 @@
 import logging
 
-from flask_login import current_user
-from flask_restful import marshal_with, reqparse
-from flask_restful.inputs import int_range
+from flask_login import current_user  # type: ignore
+from flask_restful import marshal_with, reqparse  # type: ignore
+from flask_restful.inputs import int_range  # type: ignore
 from werkzeug.exceptions import InternalServerError, NotFound
 
 import services
-from controllers.console import api
 from controllers.console.app.error import (
     AppMoreLikeThisDisabledError,
     CompletionRequestError,
@@ -67,10 +66,17 @@ class MessageFeedbackApi(InstalledAppResource):
 
         parser = reqparse.RequestParser()
         parser.add_argument("rating", type=str, choices=["like", "dislike", None], location="json")
+        parser.add_argument("content", type=str, location="json")
         args = parser.parse_args()
 
         try:
-            MessageService.create_feedback(app_model, message_id, current_user, args["rating"])
+            MessageService.create_feedback(
+                app_model=app_model,
+                message_id=message_id,
+                user=current_user,
+                rating=args.get("rating"),
+                content=args.get("content"),
+            )
         except services.errors.message.MessageNotExistsError:
             raise NotFound("Message Not Exists.")
 
@@ -153,21 +159,3 @@ class MessageSuggestedQuestionApi(InstalledAppResource):
             raise InternalServerError()
 
         return {"data": questions}
-
-
-api.add_resource(MessageListApi, "/installed-apps/<uuid:installed_app_id>/messages", endpoint="installed_app_messages")
-api.add_resource(
-    MessageFeedbackApi,
-    "/installed-apps/<uuid:installed_app_id>/messages/<uuid:message_id>/feedbacks",
-    endpoint="installed_app_message_feedback",
-)
-api.add_resource(
-    MessageMoreLikeThisApi,
-    "/installed-apps/<uuid:installed_app_id>/messages/<uuid:message_id>/more-like-this",
-    endpoint="installed_app_more_like_this",
-)
-api.add_resource(
-    MessageSuggestedQuestionApi,
-    "/installed-apps/<uuid:installed_app_id>/messages/<uuid:message_id>/suggested-questions",
-    endpoint="installed_app_suggested_question",
-)

api/controllers/console/explore/parameter.py

@@ -1,4 +1,4 @@
-from flask_restful import marshal_with
+from flask_restful import marshal_with  # type: ignore
 
 from controllers.common import fields
 from controllers.common import helpers as controller_helpers

api/controllers/console/explore/recommended_app.py

@@ -1,9 +1,10 @@
-from flask_login import current_user
-from flask_restful import Resource, fields, marshal_with, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, fields, marshal_with, reqparse  # type: ignore
 
 from constants.languages import languages
 from controllers.console import api
 from controllers.console.wraps import account_initialization_required
+from libs.helper import AppIconUrlField
 from libs.login import login_required
 from services.recommended_app_service import RecommendedAppService
 
@@ -12,6 +13,8 @@ app_fields = {
     "name": fields.String,
     "mode": fields.String,
     "icon": fields.String,
+    "icon_type": fields.String,
+    "icon_url": AppIconUrlField,
     "icon_background": fields.String,
 }
 

api/controllers/console/explore/saved_message.py

@@ -1,6 +1,6 @@
-from flask_login import current_user
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
+from flask_login import current_user  # type: ignore
+from flask_restful import fields, marshal_with, reqparse  # type: ignore
+from flask_restful.inputs import int_range  # type: ignore
 from werkzeug.exceptions import NotFound
 
 from controllers.console import api

api/controllers/console/explore/workflow.py

@@ -1,9 +1,8 @@
 import logging
 
-from flask_restful import reqparse
+from flask_restful import reqparse  # type: ignore
 from werkzeug.exceptions import InternalServerError
 
-from controllers.console import api
 from controllers.console.app.error import (
     CompletionRequestError,
     ProviderModelCurrentlyNotSupportError,
@@ -14,7 +13,11 @@ from controllers.console.explore.error import NotWorkflowAppError
 from controllers.console.explore.wraps import InstalledAppResource
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom
-from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
+from core.errors.error import (
+    ModelCurrentlyNotSupportError,
+    ProviderTokenNotInitError,
+    QuotaExceededError,
+)
 from core.model_runtime.errors.invoke import InvokeError
 from libs import helper
 from libs.login import current_user
@@ -73,9 +76,3 @@ class InstalledAppWorkflowTaskStopApi(InstalledAppResource):
         AppQueueManager.set_stop_flag(task_id, InvokeFrom.EXPLORE, current_user.id)
 
         return {"result": "success"}
-
-
-api.add_resource(InstalledAppWorkflowRunApi, "/installed-apps/<uuid:installed_app_id>/workflows/run")
-api.add_resource(
-    InstalledAppWorkflowTaskStopApi, "/installed-apps/<uuid:installed_app_id>/workflows/tasks/<string:task_id>/stop"
-)

api/controllers/console/explore/wraps.py

@@ -1,7 +1,7 @@
 from functools import wraps
 
-from flask_login import current_user
-from flask_restful import Resource
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource  # type: ignore
 from werkzeug.exceptions import NotFound
 
 from controllers.console.wraps import account_initialization_required

api/controllers/console/extension.py

@@ -1,5 +1,5 @@
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse  # type: ignore
 
 from constants import HIDDEN_VALUE
 from controllers.console import api

api/controllers/console/feature.py

@@ -1,5 +1,5 @@
-from flask_login import current_user
-from flask_restful import Resource
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource  # type: ignore
 
 from libs.login import login_required
 from services.feature_service import FeatureService

api/controllers/console/files.py

@@ -1,6 +1,9 @@
+from typing import Literal
+
 from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal_with
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, marshal_with  # type: ignore
+from werkzeug.exceptions import Forbidden
 
 import services
 from configs import dify_config
@@ -47,7 +50,8 @@ class FileApi(Resource):
     @cloud_edition_billing_resource_check("documents")
     def post(self):
         file = request.files["file"]
-        source = request.form.get("source")
+        source_str = request.form.get("source")
+        source: Literal["datasets"] | None = "datasets" if source_str == "datasets" else None
 
         if "file" not in request.files:
             raise NoFileUploadedError()
@@ -58,6 +62,9 @@ class FileApi(Resource):
         if not file.filename:
             raise FilenameNotExistsError
 
+        if source == "datasets" and not current_user.is_dataset_editor:
+            raise Forbidden()
+
         if source not in ("datasets", None):
             source = None
 

api/controllers/console/init_validate.py

@@ -1,9 +1,12 @@
 import os
 
 from flask import session
-from flask_restful import Resource, reqparse
+from flask_restful import Resource, reqparse  # type: ignore
+from sqlalchemy import select
+from sqlalchemy.orm import Session
 
 from configs import dify_config
+from extensions.ext_database import db
 from libs.helper import StrLen
 from models.model import DifySetup
 from services.account_service import TenantService
@@ -42,7 +45,11 @@ class InitValidateAPI(Resource):
 def get_init_validate_status():
     if dify_config.EDITION == "SELF_HOSTED":
         if os.environ.get("INIT_PASSWORD"):
-            return session.get("is_init_validated") or DifySetup.query.first()
+            if session.get("is_init_validated"):
+                return True
+
+            with Session(db.engine) as db_session:
+                return db_session.execute(select(DifySetup)).scalar_one_or_none()
 
     return True
 

api/controllers/console/ping.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource
+from flask_restful import Resource  # type: ignore
 
 from controllers.console import api
 

api/controllers/console/remote_files.py

@@ -2,11 +2,12 @@ import urllib.parse
 from typing import cast
 
 import httpx
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse  # type: ignore
 
 import services
 from controllers.common import helpers
+from controllers.common.errors import RemoteFileUploadError
 from core.file import helpers as file_helpers
 from core.helper import ssrf_proxy
 from fields.file_fields import file_fields_with_signed_url, remote_file_info_fields
@@ -43,10 +44,14 @@ class RemoteFileUploadApi(Resource):
 
         url = args["url"]
 
-        resp = ssrf_proxy.head(url=url)
-        if resp.status_code != httpx.codes.OK:
-            resp = ssrf_proxy.get(url=url, timeout=3, follow_redirects=True)
-        resp.raise_for_status()
+        try:
+            resp = ssrf_proxy.head(url=url)
+            if resp.status_code != httpx.codes.OK:
+                resp = ssrf_proxy.get(url=url, timeout=3, follow_redirects=True)
+            if resp.status_code != httpx.codes.OK:
+                raise RemoteFileUploadError(f"Failed to fetch file from {url}: {resp.text}")
+        except httpx.RequestError as e:
+            raise RemoteFileUploadError(f"Failed to fetch file from {url}: {str(e)}")
 
         file_info = helpers.guess_file_info_from_response(resp)
 

api/controllers/console/setup.py

@@ -1,10 +1,10 @@
 from flask import request
-from flask_restful import Resource, reqparse
+from flask_restful import Resource, reqparse  # type: ignore
 
 from configs import dify_config
 from libs.helper import StrLen, email, extract_remote_ip
 from libs.password import valid_password
-from models.model import DifySetup
+from models.model import DifySetup, db
 from services.account_service import RegisterService, TenantService
 
 from . import api
@@ -52,8 +52,9 @@ class SetupApi(Resource):
 
 def get_setup_status():
     if dify_config.EDITION == "SELF_HOSTED":
-        return DifySetup.query.first()
-    return True
+        return db.session.query(DifySetup).first()
+    else:
+        return True
 
 
 api.add_resource(SetupApi, "/setup")

api/controllers/console/tag/tags.py

@@ -1,6 +1,6 @@
 from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse  # type: ignore
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
@@ -23,7 +23,7 @@ class TagListApi(Resource):
     @account_initialization_required
     @marshal_with(tag_fields)
     def get(self):
-        tag_type = request.args.get("type", type=str)
+        tag_type = request.args.get("type", type=str, default="")
         keyword = request.args.get("keyword", default=None, type=str)
         tags = TagService.get_tags(tag_type, current_user.current_tenant_id, keyword)
 

api/controllers/console/version.py

@@ -2,7 +2,7 @@ import json
 import logging
 
 import requests
-from flask_restful import Resource, reqparse
+from flask_restful import Resource, reqparse  # type: ignore
 from packaging import version
 
 from configs import dify_config

api/controllers/console/workspace/__init__.py

@@ -0,0 +1,56 @@
+from functools import wraps
+
+from flask_login import current_user  # type: ignore
+from sqlalchemy.orm import Session
+from werkzeug.exceptions import Forbidden
+
+from extensions.ext_database import db
+from models.account import TenantPluginPermission
+
+
+def plugin_permission_required(
+    install_required: bool = False,
+    debug_required: bool = False,
+):
+    def interceptor(view):
+        @wraps(view)
+        def decorated(*args, **kwargs):
+            user = current_user
+            tenant_id = user.current_tenant_id
+
+            with Session(db.engine) as session:
+                permission = (
+                    session.query(TenantPluginPermission)
+                    .filter(
+                        TenantPluginPermission.tenant_id == tenant_id,
+                    )
+                    .first()
+                )
+
+                if not permission:
+                    # no permission set, allow access for everyone
+                    return view(*args, **kwargs)
+
+                if install_required:
+                    if permission.install_permission == TenantPluginPermission.InstallPermission.NOBODY:
+                        raise Forbidden()
+                    if permission.install_permission == TenantPluginPermission.InstallPermission.ADMINS:
+                        if not user.is_admin_or_owner:
+                            raise Forbidden()
+                    if permission.install_permission == TenantPluginPermission.InstallPermission.EVERYONE:
+                        pass
+
+                if debug_required:
+                    if permission.debug_permission == TenantPluginPermission.DebugPermission.NOBODY:
+                        raise Forbidden()
+                    if permission.debug_permission == TenantPluginPermission.DebugPermission.ADMINS:
+                        if not user.is_admin_or_owner:
+                            raise Forbidden()
+                    if permission.debug_permission == TenantPluginPermission.DebugPermission.EVERYONE:
+                        pass
+
+            return view(*args, **kwargs)
+
+        return decorated
+
+    return interceptor

api/controllers/console/workspace/account.py

@@ -2,8 +2,8 @@ import datetime
 
 import pytz
 from flask import request
-from flask_login import current_user
-from flask_restful import Resource, fields, marshal_with, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, fields, marshal_with, reqparse  # type: ignore
 
 from configs import dify_config
 from constants.languages import supported_language
@@ -11,6 +11,7 @@ from controllers.console import api
 from controllers.console.workspace.error import (
     AccountAlreadyInitedError,
     CurrentPasswordIncorrectError,
+    InvalidAccountDeletionCodeError,
     InvalidInvitationCodeError,
     RepeatPasswordNotMatchError,
 )
@@ -21,6 +22,7 @@ from libs.helper import TimestampField, timezone
 from libs.login import login_required
 from models import AccountIntegrate, InvitationCode
 from services.account_service import AccountService
+from services.billing_service import BillingService
 from services.errors.account import CurrentPasswordIncorrectError as ServiceCurrentPasswordIncorrectError
 
 
@@ -242,6 +244,54 @@ class AccountIntegrateApi(Resource):
         return {"data": integrate_data}
 
 
+class AccountDeleteVerifyApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        account = current_user
+
+        token, code = AccountService.generate_account_deletion_verification_code(account)
+        AccountService.send_account_deletion_verification_email(account, code)
+
+        return {"result": "success", "data": token}
+
+
+class AccountDeleteApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        account = current_user
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("token", type=str, required=True, location="json")
+        parser.add_argument("code", type=str, required=True, location="json")
+        args = parser.parse_args()
+
+        if not AccountService.verify_account_deletion_code(args["token"], args["code"]):
+            raise InvalidAccountDeletionCodeError()
+
+        AccountService.delete_account(account)
+
+        return {"result": "success"}
+
+
+class AccountDeleteUpdateFeedbackApi(Resource):
+    @setup_required
+    def post(self):
+        account = current_user
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("email", type=str, required=True, location="json")
+        parser.add_argument("feedback", type=str, required=True, location="json")
+        args = parser.parse_args()
+
+        BillingService.update_account_deletion_feedback(args["email"], args["feedback"])
+
+        return {"result": "success"}
+
+
 # Register API resources
 api.add_resource(AccountInitApi, "/account/init")
 api.add_resource(AccountProfileApi, "/account/profile")
@@ -252,5 +302,8 @@ api.add_resource(AccountInterfaceThemeApi, "/account/interface-theme")
 api.add_resource(AccountTimezoneApi, "/account/timezone")
 api.add_resource(AccountPasswordApi, "/account/password")
 api.add_resource(AccountIntegrateApi, "/account/integrates")
+api.add_resource(AccountDeleteVerifyApi, "/account/delete/verify")
+api.add_resource(AccountDeleteApi, "/account/delete")
+api.add_resource(AccountDeleteUpdateFeedbackApi, "/account/delete/feedback")
 # api.add_resource(AccountEmailApi, '/account/email')
 # api.add_resource(AccountEmailVerifyApi, '/account/email-verify')

api/controllers/console/workspace/agent_providers.py

@@ -0,0 +1,36 @@
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource  # type: ignore
+
+from controllers.console import api
+from controllers.console.wraps import account_initialization_required, setup_required
+from core.model_runtime.utils.encoders import jsonable_encoder
+from libs.login import login_required
+from services.agent_service import AgentService
+
+
+class AgentProviderListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        user = current_user
+
+        user_id = user.id
+        tenant_id = user.current_tenant_id
+
+        return jsonable_encoder(AgentService.list_agent_providers(user_id, tenant_id))
+
+
+class AgentProviderApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, provider_name: str):
+        user = current_user
+        user_id = user.id
+        tenant_id = user.current_tenant_id
+        return jsonable_encoder(AgentService.get_agent_provider(user_id, tenant_id, provider_name))
+
+
+api.add_resource(AgentProviderListApi, "/workspaces/current/agent-providers")
+api.add_resource(AgentProviderApi, "/workspaces/current/agent-provider/<path:provider_name>")

api/controllers/console/workspace/endpoint.py

@@ -0,0 +1,205 @@
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, reqparse  # type: ignore
+from werkzeug.exceptions import Forbidden
+
+from controllers.console import api
+from controllers.console.wraps import account_initialization_required, setup_required
+from core.model_runtime.utils.encoders import jsonable_encoder
+from libs.login import login_required
+from services.plugin.endpoint_service import EndpointService
+
+
+class EndpointCreateApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        user = current_user
+        if not user.is_admin_or_owner:
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("plugin_unique_identifier", type=str, required=True)
+        parser.add_argument("settings", type=dict, required=True)
+        parser.add_argument("name", type=str, required=True)
+        args = parser.parse_args()
+
+        plugin_unique_identifier = args["plugin_unique_identifier"]
+        settings = args["settings"]
+        name = args["name"]
+
+        return {
+            "success": EndpointService.create_endpoint(
+                tenant_id=user.current_tenant_id,
+                user_id=user.id,
+                plugin_unique_identifier=plugin_unique_identifier,
+                name=name,
+                settings=settings,
+            )
+        }
+
+
+class EndpointListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        user = current_user
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("page", type=int, required=True, location="args")
+        parser.add_argument("page_size", type=int, required=True, location="args")
+        args = parser.parse_args()
+
+        page = args["page"]
+        page_size = args["page_size"]
+
+        return jsonable_encoder(
+            {
+                "endpoints": EndpointService.list_endpoints(
+                    tenant_id=user.current_tenant_id,
+                    user_id=user.id,
+                    page=page,
+                    page_size=page_size,
+                )
+            }
+        )
+
+
+class EndpointListForSinglePluginApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        user = current_user
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("page", type=int, required=True, location="args")
+        parser.add_argument("page_size", type=int, required=True, location="args")
+        parser.add_argument("plugin_id", type=str, required=True, location="args")
+        args = parser.parse_args()
+
+        page = args["page"]
+        page_size = args["page_size"]
+        plugin_id = args["plugin_id"]
+
+        return jsonable_encoder(
+            {
+                "endpoints": EndpointService.list_endpoints_for_single_plugin(
+                    tenant_id=user.current_tenant_id,
+                    user_id=user.id,
+                    plugin_id=plugin_id,
+                    page=page,
+                    page_size=page_size,
+                )
+            }
+        )
+
+
+class EndpointDeleteApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        user = current_user
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("endpoint_id", type=str, required=True)
+        args = parser.parse_args()
+
+        if not user.is_admin_or_owner:
+            raise Forbidden()
+
+        endpoint_id = args["endpoint_id"]
+
+        return {
+            "success": EndpointService.delete_endpoint(
+                tenant_id=user.current_tenant_id, user_id=user.id, endpoint_id=endpoint_id
+            )
+        }
+
+
+class EndpointUpdateApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        user = current_user
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("endpoint_id", type=str, required=True)
+        parser.add_argument("settings", type=dict, required=True)
+        parser.add_argument("name", type=str, required=True)
+        args = parser.parse_args()
+
+        endpoint_id = args["endpoint_id"]
+        settings = args["settings"]
+        name = args["name"]
+
+        if not user.is_admin_or_owner:
+            raise Forbidden()
+
+        return {
+            "success": EndpointService.update_endpoint(
+                tenant_id=user.current_tenant_id,
+                user_id=user.id,
+                endpoint_id=endpoint_id,
+                name=name,
+                settings=settings,
+            )
+        }
+
+
+class EndpointEnableApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        user = current_user
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("endpoint_id", type=str, required=True)
+        args = parser.parse_args()
+
+        endpoint_id = args["endpoint_id"]
+
+        if not user.is_admin_or_owner:
+            raise Forbidden()
+
+        return {
+            "success": EndpointService.enable_endpoint(
+                tenant_id=user.current_tenant_id, user_id=user.id, endpoint_id=endpoint_id
+            )
+        }
+
+
+class EndpointDisableApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        user = current_user
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("endpoint_id", type=str, required=True)
+        args = parser.parse_args()
+
+        endpoint_id = args["endpoint_id"]
+
+        if not user.is_admin_or_owner:
+            raise Forbidden()
+
+        return {
+            "success": EndpointService.disable_endpoint(
+                tenant_id=user.current_tenant_id, user_id=user.id, endpoint_id=endpoint_id
+            )
+        }
+
+
+api.add_resource(EndpointCreateApi, "/workspaces/current/endpoints/create")
+api.add_resource(EndpointListApi, "/workspaces/current/endpoints/list")
+api.add_resource(EndpointListForSinglePluginApi, "/workspaces/current/endpoints/list/plugin")
+api.add_resource(EndpointDeleteApi, "/workspaces/current/endpoints/delete")
+api.add_resource(EndpointUpdateApi, "/workspaces/current/endpoints/update")
+api.add_resource(EndpointEnableApi, "/workspaces/current/endpoints/enable")
+api.add_resource(EndpointDisableApi, "/workspaces/current/endpoints/disable")

api/controllers/console/workspace/error.py

@@ -35,3 +35,9 @@ class AccountNotInitializedError(BaseHTTPException):
     error_code = "account_not_initialized"
     description = "The account has not been initialized yet. Please proceed with the initialization process first."
     code = 400
+
+
+class InvalidAccountDeletionCodeError(BaseHTTPException):
+    error_code = "invalid_account_deletion_code"
+    description = "Invalid account deletion code."
+    code = 400

api/controllers/console/workspace/load_balancing_config.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource, reqparse
+from flask_restful import Resource, reqparse  # type: ignore
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
@@ -37,7 +37,7 @@ class LoadBalancingCredentialsValidateApi(Resource):
         model_load_balancing_service = ModelLoadBalancingService()
 
         result = True
-        error = None
+        error = ""
 
         try:
             model_load_balancing_service.validate_load_balancing_credentials(
@@ -86,7 +86,7 @@ class LoadBalancingConfigCredentialsValidateApi(Resource):
         model_load_balancing_service = ModelLoadBalancingService()
 
         result = True
-        error = None
+        error = ""
 
         try:
             model_load_balancing_service.validate_load_balancing_credentials(
@@ -112,10 +112,10 @@ class LoadBalancingConfigCredentialsValidateApi(Resource):
 # Load Balancing Config
 api.add_resource(
     LoadBalancingCredentialsValidateApi,
-    "/workspaces/current/model-providers/<string:provider>/models/load-balancing-configs/credentials-validate",
+    "/workspaces/current/model-providers/<path:provider>/models/load-balancing-configs/credentials-validate",
 )
 
 api.add_resource(
     LoadBalancingConfigCredentialsValidateApi,
-    "/workspaces/current/model-providers/<string:provider>/models/load-balancing-configs/<string:config_id>/credentials-validate",
+    "/workspaces/current/model-providers/<path:provider>/models/load-balancing-configs/<string:config_id>/credentials-validate",
 )

api/controllers/console/workspace/members.py

@@ -1,7 +1,7 @@
 from urllib import parse
 
-from flask_login import current_user
-from flask_restful import Resource, abort, marshal_with, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, abort, marshal_with, reqparse  # type: ignore
 
 import services
 from configs import dify_config
@@ -89,19 +89,19 @@ class MemberCancelInviteApi(Resource):
     @account_initialization_required
     def delete(self, member_id):
         member = db.session.query(Account).filter(Account.id == str(member_id)).first()
-        if not member:
+        if member is None:
             abort(404)
-
-        try:
-            TenantService.remove_member_from_tenant(current_user.current_tenant, member, current_user)
-        except services.errors.account.CannotOperateSelfError as e:
-            return {"code": "cannot-operate-self", "message": str(e)}, 400
-        except services.errors.account.NoPermissionError as e:
-            return {"code": "forbidden", "message": str(e)}, 403
-        except services.errors.account.MemberNotInTenantError as e:
-            return {"code": "member-not-found", "message": str(e)}, 404
-        except Exception as e:
-            raise ValueError(str(e))
+        else:
+            try:
+                TenantService.remove_member_from_tenant(current_user.current_tenant, member, current_user)
+            except services.errors.account.CannotOperateSelfError as e:
+                return {"code": "cannot-operate-self", "message": str(e)}, 400
+            except services.errors.account.NoPermissionError as e:
+                return {"code": "forbidden", "message": str(e)}, 403
+            except services.errors.account.MemberNotInTenantError as e:
+                return {"code": "member-not-found", "message": str(e)}, 404
+            except Exception as e:
+                raise ValueError(str(e))
 
         return {"result": "success"}, 204
 
@@ -126,6 +126,7 @@ class MemberUpdateRoleApi(Resource):
             abort(404)
 
         try:
+            assert member is not None, "Member not found"
             TenantService.update_member_role(current_user.current_tenant, member, new_role, current_user)
         except Exception as e:
             raise ValueError(str(e))

api/controllers/console/workspace/model_providers.py

@@ -1,8 +1,8 @@
 import io
 
 from flask import send_file
-from flask_login import current_user
-from flask_restful import Resource, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, reqparse  # type: ignore
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
@@ -66,7 +66,7 @@ class ModelProviderValidateApi(Resource):
         model_provider_service = ModelProviderService()
 
         result = True
-        error = None
+        error = ""
 
         try:
             model_provider_service.provider_credentials_validate(
@@ -79,7 +79,7 @@ class ModelProviderValidateApi(Resource):
         response = {"result": "success" if result else "error"}
 
         if not result:
-            response["error"] = error
+            response["error"] = error or "Unknown error"
 
         return response
 
@@ -125,14 +125,16 @@ class ModelProviderIconApi(Resource):
     Get model provider icon
     """
 
-    def get(self, provider: str, icon_type: str, lang: str):
+    def get(self, tenant_id: str, provider: str, icon_type: str, lang: str):
         model_provider_service = ModelProviderService()
         icon, mimetype = model_provider_service.get_model_provider_icon(
+            tenant_id=tenant_id,
             provider=provider,
             icon_type=icon_type,
             lang=lang,
         )
-
+        if icon is None:
+            raise ValueError(f"icon not found for provider {provider}, icon_type {icon_type}, lang {lang}")
         return send_file(io.BytesIO(icon), mimetype=mimetype)
 
 
@@ -182,53 +184,17 @@ class ModelProviderPaymentCheckoutUrlApi(Resource):
         return data
 
 
-class ModelProviderFreeQuotaSubmitApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def post(self, provider: str):
-        model_provider_service = ModelProviderService()
-        result = model_provider_service.free_quota_submit(tenant_id=current_user.current_tenant_id, provider=provider)
-
-        return result
-
-
-class ModelProviderFreeQuotaQualificationVerifyApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def get(self, provider: str):
-        parser = reqparse.RequestParser()
-        parser.add_argument("token", type=str, required=False, nullable=True, location="args")
-        args = parser.parse_args()
-
-        model_provider_service = ModelProviderService()
-        result = model_provider_service.free_quota_qualification_verify(
-            tenant_id=current_user.current_tenant_id, provider=provider, token=args["token"]
-        )
-
-        return result
-
-
 api.add_resource(ModelProviderListApi, "/workspaces/current/model-providers")
 
-api.add_resource(ModelProviderCredentialApi, "/workspaces/current/model-providers/<string:provider>/credentials")
-api.add_resource(ModelProviderValidateApi, "/workspaces/current/model-providers/<string:provider>/credentials/validate")
-api.add_resource(ModelProviderApi, "/workspaces/current/model-providers/<string:provider>")
-api.add_resource(
-    ModelProviderIconApi, "/workspaces/current/model-providers/<string:provider>/<string:icon_type>/<string:lang>"
-)
+api.add_resource(ModelProviderCredentialApi, "/workspaces/current/model-providers/<path:provider>/credentials")
+api.add_resource(ModelProviderValidateApi, "/workspaces/current/model-providers/<path:provider>/credentials/validate")
+api.add_resource(ModelProviderApi, "/workspaces/current/model-providers/<path:provider>")
 
 api.add_resource(
-    PreferredProviderTypeUpdateApi, "/workspaces/current/model-providers/<string:provider>/preferred-provider-type"
+    PreferredProviderTypeUpdateApi, "/workspaces/current/model-providers/<path:provider>/preferred-provider-type"
 )
+api.add_resource(ModelProviderPaymentCheckoutUrlApi, "/workspaces/current/model-providers/<path:provider>/checkout-url")
 api.add_resource(
-    ModelProviderPaymentCheckoutUrlApi, "/workspaces/current/model-providers/<string:provider>/checkout-url"
-)
-api.add_resource(
-    ModelProviderFreeQuotaSubmitApi, "/workspaces/current/model-providers/<string:provider>/free-quota-submit"
-)
-api.add_resource(
-    ModelProviderFreeQuotaQualificationVerifyApi,
-    "/workspaces/current/model-providers/<string:provider>/free-quota-qualification-verify",
+    ModelProviderIconApi,
+    "/workspaces/<string:tenant_id>/model-providers/<path:provider>/<string:icon_type>/<string:lang>",
 )

api/controllers/console/workspace/models.py

@@ -1,7 +1,7 @@
 import logging
 
-from flask_login import current_user
-from flask_restful import Resource, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, reqparse  # type: ignore
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
@@ -308,7 +308,7 @@ class ModelProviderModelValidateApi(Resource):
         model_provider_service = ModelProviderService()
 
         result = True
-        error = None
+        error = ""
 
         try:
             model_provider_service.model_credentials_validate(
@@ -325,7 +325,7 @@ class ModelProviderModelValidateApi(Resource):
         response = {"result": "success" if result else "error"}
 
         if not result:
-            response["error"] = error
+            response["error"] = error or ""
 
         return response
 
@@ -362,26 +362,26 @@ class ModelProviderAvailableModelApi(Resource):
         return jsonable_encoder({"data": models})
 
 
-api.add_resource(ModelProviderModelApi, "/workspaces/current/model-providers/<string:provider>/models")
+api.add_resource(ModelProviderModelApi, "/workspaces/current/model-providers/<path:provider>/models")
 api.add_resource(
     ModelProviderModelEnableApi,
-    "/workspaces/current/model-providers/<string:provider>/models/enable",
+    "/workspaces/current/model-providers/<path:provider>/models/enable",
     endpoint="model-provider-model-enable",
 )
 api.add_resource(
     ModelProviderModelDisableApi,
-    "/workspaces/current/model-providers/<string:provider>/models/disable",
+    "/workspaces/current/model-providers/<path:provider>/models/disable",
     endpoint="model-provider-model-disable",
 )
 api.add_resource(
-    ModelProviderModelCredentialApi, "/workspaces/current/model-providers/<string:provider>/models/credentials"
+    ModelProviderModelCredentialApi, "/workspaces/current/model-providers/<path:provider>/models/credentials"
 )
 api.add_resource(
-    ModelProviderModelValidateApi, "/workspaces/current/model-providers/<string:provider>/models/credentials/validate"
+    ModelProviderModelValidateApi, "/workspaces/current/model-providers/<path:provider>/models/credentials/validate"
 )
 
 api.add_resource(
-    ModelProviderModelParameterRuleApi, "/workspaces/current/model-providers/<string:provider>/models/parameter-rules"
+    ModelProviderModelParameterRuleApi, "/workspaces/current/model-providers/<path:provider>/models/parameter-rules"
 )
 api.add_resource(ModelProviderAvailableModelApi, "/workspaces/current/models/model-types/<string:model_type>")
 api.add_resource(DefaultModelApi, "/workspaces/current/default-model")

api/controllers/console/workspace/plugin.py

@@ -0,0 +1,475 @@
+import io
+
+from flask import request, send_file
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, reqparse  # type: ignore
+from werkzeug.exceptions import Forbidden
+
+from configs import dify_config
+from controllers.console import api
+from controllers.console.workspace import plugin_permission_required
+from controllers.console.wraps import account_initialization_required, setup_required
+from core.model_runtime.utils.encoders import jsonable_encoder
+from core.plugin.manager.exc import PluginDaemonClientSideError
+from libs.login import login_required
+from models.account import TenantPluginPermission
+from services.plugin.plugin_permission_service import PluginPermissionService
+from services.plugin.plugin_service import PluginService
+
+
+class PluginDebuggingKeyApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @plugin_permission_required(debug_required=True)
+    def get(self):
+        tenant_id = current_user.current_tenant_id
+
+        try:
+            return {
+                "key": PluginService.get_debugging_key(tenant_id),
+                "host": dify_config.PLUGIN_REMOTE_INSTALL_HOST,
+                "port": dify_config.PLUGIN_REMOTE_INSTALL_PORT,
+            }
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+
+class PluginListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        tenant_id = current_user.current_tenant_id
+        try:
+            plugins = PluginService.list(tenant_id)
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+        return jsonable_encoder({"plugins": plugins})
+
+
+class PluginListInstallationsFromIdsApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        tenant_id = current_user.current_tenant_id
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("plugin_ids", type=list, required=True, location="json")
+        args = parser.parse_args()
+
+        try:
+            plugins = PluginService.list_installations_from_ids(tenant_id, args["plugin_ids"])
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+        return jsonable_encoder({"plugins": plugins})
+
+
+class PluginIconApi(Resource):
+    @setup_required
+    def get(self):
+        req = reqparse.RequestParser()
+        req.add_argument("tenant_id", type=str, required=True, location="args")
+        req.add_argument("filename", type=str, required=True, location="args")
+        args = req.parse_args()
+
+        try:
+            icon_bytes, mimetype = PluginService.get_asset(args["tenant_id"], args["filename"])
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+        icon_cache_max_age = dify_config.TOOL_ICON_CACHE_MAX_AGE
+        return send_file(io.BytesIO(icon_bytes), mimetype=mimetype, max_age=icon_cache_max_age)
+
+
+class PluginUploadFromPkgApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @plugin_permission_required(install_required=True)
+    def post(self):
+        tenant_id = current_user.current_tenant_id
+
+        file = request.files["pkg"]
+
+        # check file size
+        if file.content_length > dify_config.PLUGIN_MAX_PACKAGE_SIZE:
+            raise ValueError("File size exceeds the maximum allowed size")
+
+        content = file.read()
+        try:
+            response = PluginService.upload_pkg(tenant_id, content)
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+        return jsonable_encoder(response)
+
+
+class PluginUploadFromGithubApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @plugin_permission_required(install_required=True)
+    def post(self):
+        tenant_id = current_user.current_tenant_id
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("repo", type=str, required=True, location="json")
+        parser.add_argument("version", type=str, required=True, location="json")
+        parser.add_argument("package", type=str, required=True, location="json")
+        args = parser.parse_args()
+
+        try:
+            response = PluginService.upload_pkg_from_github(tenant_id, args["repo"], args["version"], args["package"])
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+        return jsonable_encoder(response)
+
+
+class PluginUploadFromBundleApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @plugin_permission_required(install_required=True)
+    def post(self):
+        tenant_id = current_user.current_tenant_id
+
+        file = request.files["bundle"]
+
+        # check file size
+        if file.content_length > dify_config.PLUGIN_MAX_BUNDLE_SIZE:
+            raise ValueError("File size exceeds the maximum allowed size")
+
+        content = file.read()
+        try:
+            response = PluginService.upload_bundle(tenant_id, content)
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+        return jsonable_encoder(response)
+
+
+class PluginInstallFromPkgApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @plugin_permission_required(install_required=True)
+    def post(self):
+        tenant_id = current_user.current_tenant_id
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("plugin_unique_identifiers", type=list, required=True, location="json")
+        args = parser.parse_args()
+
+        # check if all plugin_unique_identifiers are valid string
+        for plugin_unique_identifier in args["plugin_unique_identifiers"]:
+            if not isinstance(plugin_unique_identifier, str):
+                raise ValueError("Invalid plugin unique identifier")
+
+        try:
+            response = PluginService.install_from_local_pkg(tenant_id, args["plugin_unique_identifiers"])
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+        return jsonable_encoder(response)
+
+
+class PluginInstallFromGithubApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @plugin_permission_required(install_required=True)
+    def post(self):
+        tenant_id = current_user.current_tenant_id
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("repo", type=str, required=True, location="json")
+        parser.add_argument("version", type=str, required=True, location="json")
+        parser.add_argument("package", type=str, required=True, location="json")
+        parser.add_argument("plugin_unique_identifier", type=str, required=True, location="json")
+        args = parser.parse_args()
+
+        try:
+            response = PluginService.install_from_github(
+                tenant_id,
+                args["plugin_unique_identifier"],
+                args["repo"],
+                args["version"],
+                args["package"],
+            )
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+        return jsonable_encoder(response)
+
+
+class PluginInstallFromMarketplaceApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @plugin_permission_required(install_required=True)
+    def post(self):
+        tenant_id = current_user.current_tenant_id
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("plugin_unique_identifiers", type=list, required=True, location="json")
+        args = parser.parse_args()
+
+        # check if all plugin_unique_identifiers are valid string
+        for plugin_unique_identifier in args["plugin_unique_identifiers"]:
+            if not isinstance(plugin_unique_identifier, str):
+                raise ValueError("Invalid plugin unique identifier")
+
+        try:
+            response = PluginService.install_from_marketplace_pkg(tenant_id, args["plugin_unique_identifiers"])
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+        return jsonable_encoder(response)
+
+
+class PluginFetchManifestApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @plugin_permission_required(debug_required=True)
+    def get(self):
+        tenant_id = current_user.current_tenant_id
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("plugin_unique_identifier", type=str, required=True, location="args")
+        args = parser.parse_args()
+
+        try:
+            return jsonable_encoder(
+                {
+                    "manifest": PluginService.fetch_plugin_manifest(
+                        tenant_id, args["plugin_unique_identifier"]
+                    ).model_dump()
+                }
+            )
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+
+class PluginFetchInstallTasksApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @plugin_permission_required(debug_required=True)
+    def get(self):
+        tenant_id = current_user.current_tenant_id
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("page", type=int, required=True, location="args")
+        parser.add_argument("page_size", type=int, required=True, location="args")
+        args = parser.parse_args()
+
+        try:
+            return jsonable_encoder(
+                {"tasks": PluginService.fetch_install_tasks(tenant_id, args["page"], args["page_size"])}
+            )
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+
+class PluginFetchInstallTaskApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @plugin_permission_required(debug_required=True)
+    def get(self, task_id: str):
+        tenant_id = current_user.current_tenant_id
+
+        try:
+            return jsonable_encoder({"task": PluginService.fetch_install_task(tenant_id, task_id)})
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+
+class PluginDeleteInstallTaskApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @plugin_permission_required(debug_required=True)
+    def post(self, task_id: str):
+        tenant_id = current_user.current_tenant_id
+
+        try:
+            return {"success": PluginService.delete_install_task(tenant_id, task_id)}
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+
+class PluginDeleteAllInstallTaskItemsApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @plugin_permission_required(debug_required=True)
+    def post(self):
+        tenant_id = current_user.current_tenant_id
+
+        try:
+            return {"success": PluginService.delete_all_install_task_items(tenant_id)}
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+
+class PluginDeleteInstallTaskItemApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @plugin_permission_required(debug_required=True)
+    def post(self, task_id: str, identifier: str):
+        tenant_id = current_user.current_tenant_id
+
+        try:
+            return {"success": PluginService.delete_install_task_item(tenant_id, task_id, identifier)}
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+
+class PluginUpgradeFromMarketplaceApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @plugin_permission_required(debug_required=True)
+    def post(self):
+        tenant_id = current_user.current_tenant_id
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("original_plugin_unique_identifier", type=str, required=True, location="json")
+        parser.add_argument("new_plugin_unique_identifier", type=str, required=True, location="json")
+        args = parser.parse_args()
+
+        try:
+            return jsonable_encoder(
+                PluginService.upgrade_plugin_with_marketplace(
+                    tenant_id, args["original_plugin_unique_identifier"], args["new_plugin_unique_identifier"]
+                )
+            )
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+
+class PluginUpgradeFromGithubApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @plugin_permission_required(debug_required=True)
+    def post(self):
+        tenant_id = current_user.current_tenant_id
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("original_plugin_unique_identifier", type=str, required=True, location="json")
+        parser.add_argument("new_plugin_unique_identifier", type=str, required=True, location="json")
+        parser.add_argument("repo", type=str, required=True, location="json")
+        parser.add_argument("version", type=str, required=True, location="json")
+        parser.add_argument("package", type=str, required=True, location="json")
+        args = parser.parse_args()
+
+        try:
+            return jsonable_encoder(
+                PluginService.upgrade_plugin_with_github(
+                    tenant_id,
+                    args["original_plugin_unique_identifier"],
+                    args["new_plugin_unique_identifier"],
+                    args["repo"],
+                    args["version"],
+                    args["package"],
+                )
+            )
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+
+class PluginUninstallApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @plugin_permission_required(debug_required=True)
+    def post(self):
+        req = reqparse.RequestParser()
+        req.add_argument("plugin_installation_id", type=str, required=True, location="json")
+        args = req.parse_args()
+
+        tenant_id = current_user.current_tenant_id
+
+        try:
+            return {"success": PluginService.uninstall(tenant_id, args["plugin_installation_id"])}
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+
+class PluginChangePermissionApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        user = current_user
+        if not user.is_admin_or_owner:
+            raise Forbidden()
+
+        req = reqparse.RequestParser()
+        req.add_argument("install_permission", type=str, required=True, location="json")
+        req.add_argument("debug_permission", type=str, required=True, location="json")
+        args = req.parse_args()
+
+        install_permission = TenantPluginPermission.InstallPermission(args["install_permission"])
+        debug_permission = TenantPluginPermission.DebugPermission(args["debug_permission"])
+
+        tenant_id = user.current_tenant_id
+
+        return {"success": PluginPermissionService.change_permission(tenant_id, install_permission, debug_permission)}
+
+
+class PluginFetchPermissionApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        tenant_id = current_user.current_tenant_id
+
+        permission = PluginPermissionService.get_permission(tenant_id)
+        if not permission:
+            return jsonable_encoder(
+                {
+                    "install_permission": TenantPluginPermission.InstallPermission.EVERYONE,
+                    "debug_permission": TenantPluginPermission.DebugPermission.EVERYONE,
+                }
+            )
+
+        return jsonable_encoder(
+            {
+                "install_permission": permission.install_permission,
+                "debug_permission": permission.debug_permission,
+            }
+        )
+
+
+api.add_resource(PluginDebuggingKeyApi, "/workspaces/current/plugin/debugging-key")
+api.add_resource(PluginListApi, "/workspaces/current/plugin/list")
+api.add_resource(PluginListInstallationsFromIdsApi, "/workspaces/current/plugin/list/installations/ids")
+api.add_resource(PluginIconApi, "/workspaces/current/plugin/icon")
+api.add_resource(PluginUploadFromPkgApi, "/workspaces/current/plugin/upload/pkg")
+api.add_resource(PluginUploadFromGithubApi, "/workspaces/current/plugin/upload/github")
+api.add_resource(PluginUploadFromBundleApi, "/workspaces/current/plugin/upload/bundle")
+api.add_resource(PluginInstallFromPkgApi, "/workspaces/current/plugin/install/pkg")
+api.add_resource(PluginInstallFromGithubApi, "/workspaces/current/plugin/install/github")
+api.add_resource(PluginUpgradeFromMarketplaceApi, "/workspaces/current/plugin/upgrade/marketplace")
+api.add_resource(PluginUpgradeFromGithubApi, "/workspaces/current/plugin/upgrade/github")
+api.add_resource(PluginInstallFromMarketplaceApi, "/workspaces/current/plugin/install/marketplace")
+api.add_resource(PluginFetchManifestApi, "/workspaces/current/plugin/fetch-manifest")
+api.add_resource(PluginFetchInstallTasksApi, "/workspaces/current/plugin/tasks")
+api.add_resource(PluginFetchInstallTaskApi, "/workspaces/current/plugin/tasks/<task_id>")
+api.add_resource(PluginDeleteInstallTaskApi, "/workspaces/current/plugin/tasks/<task_id>/delete")
+api.add_resource(PluginDeleteAllInstallTaskItemsApi, "/workspaces/current/plugin/tasks/delete_all")
+api.add_resource(PluginDeleteInstallTaskItemApi, "/workspaces/current/plugin/tasks/<task_id>/delete/<path:identifier>")
+api.add_resource(PluginUninstallApi, "/workspaces/current/plugin/uninstall")
+
+api.add_resource(PluginChangePermissionApi, "/workspaces/current/plugin/permission/change")
+api.add_resource(PluginFetchPermissionApi, "/workspaces/current/plugin/permission/fetch")

api/controllers/console/workspace/tool_providers.py

@@ -1,14 +1,16 @@
 import io
 
 from flask import send_file
-from flask_login import current_user
-from flask_restful import Resource, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, reqparse  # type: ignore
+from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden
 
 from configs import dify_config
 from controllers.console import api
 from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required
 from core.model_runtime.utils.encoders import jsonable_encoder
+from extensions.ext_database import db
 from libs.helper import alphanumeric, uuid_value
 from libs.login import login_required
 from services.tools.api_tools_manage_service import ApiToolManageService
@@ -23,8 +25,10 @@ class ToolProviderListApi(Resource):
     @login_required
     @account_initialization_required
     def get(self):
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user = current_user
+
+        user_id = user.id
+        tenant_id = user.current_tenant_id
 
         req = reqparse.RequestParser()
         req.add_argument(
@@ -45,28 +49,43 @@ class ToolBuiltinProviderListToolsApi(Resource):
     @login_required
     @account_initialization_required
     def get(self, provider):
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user = current_user
+
+        tenant_id = user.current_tenant_id
 
         return jsonable_encoder(
             BuiltinToolManageService.list_builtin_tool_provider_tools(
-                user_id,
                 tenant_id,
                 provider,
             )
         )
 
 
+class ToolBuiltinProviderInfoApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, provider):
+        user = current_user
+
+        user_id = user.id
+        tenant_id = user.current_tenant_id
+
+        return jsonable_encoder(BuiltinToolManageService.get_builtin_tool_provider_info(user_id, tenant_id, provider))
+
+
 class ToolBuiltinProviderDeleteApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
     def post(self, provider):
-        if not current_user.is_admin_or_owner:
+        user = current_user
+
+        if not user.is_admin_or_owner:
             raise Forbidden()
 
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user_id = user.id
+        tenant_id = user.current_tenant_id
 
         return BuiltinToolManageService.delete_builtin_tool_provider(
             user_id,
@@ -80,23 +99,29 @@ class ToolBuiltinProviderUpdateApi(Resource):
     @login_required
     @account_initialization_required
     def post(self, provider):
-        if not current_user.is_admin_or_owner:
+        user = current_user
+
+        if not user.is_admin_or_owner:
             raise Forbidden()
 
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user_id = user.id
+        tenant_id = user.current_tenant_id
 
         parser = reqparse.RequestParser()
         parser.add_argument("credentials", type=dict, required=True, nullable=False, location="json")
 
         args = parser.parse_args()
 
-        return BuiltinToolManageService.update_builtin_tool_provider(
-            user_id,
-            tenant_id,
-            provider,
-            args["credentials"],
-        )
+        with Session(db.engine) as session:
+            result = BuiltinToolManageService.update_builtin_tool_provider(
+                session=session,
+                user_id=user_id,
+                tenant_id=tenant_id,
+                provider_name=provider,
+                credentials=args["credentials"],
+            )
+            session.commit()
+        return result
 
 
 class ToolBuiltinProviderGetCredentialsApi(Resource):
@@ -104,13 +129,11 @@ class ToolBuiltinProviderGetCredentialsApi(Resource):
     @login_required
     @account_initialization_required
     def get(self, provider):
-        user_id = current_user.id
         tenant_id = current_user.current_tenant_id
 
         return BuiltinToolManageService.get_builtin_tool_provider_credentials(
-            user_id,
-            tenant_id,
-            provider,
+            tenant_id=tenant_id,
+            provider_name=provider,
         )
 
 
@@ -127,11 +150,13 @@ class ToolApiProviderAddApi(Resource):
     @login_required
     @account_initialization_required
     def post(self):
-        if not current_user.is_admin_or_owner:
+        user = current_user
+
+        if not user.is_admin_or_owner:
             raise Forbidden()
 
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user_id = user.id
+        tenant_id = user.current_tenant_id
 
         parser = reqparse.RequestParser()
         parser.add_argument("credentials", type=dict, required=True, nullable=False, location="json")
@@ -164,6 +189,11 @@ class ToolApiProviderGetRemoteSchemaApi(Resource):
     @login_required
     @account_initialization_required
     def get(self):
+        user = current_user
+
+        user_id = user.id
+        tenant_id = user.current_tenant_id
+
         parser = reqparse.RequestParser()
 
         parser.add_argument("url", type=str, required=True, nullable=False, location="args")
@@ -171,8 +201,8 @@ class ToolApiProviderGetRemoteSchemaApi(Resource):
         args = parser.parse_args()
 
         return ApiToolManageService.get_api_tool_provider_remote_schema(
-            current_user.id,
-            current_user.current_tenant_id,
+            user_id,
+            tenant_id,
             args["url"],
         )
 
@@ -182,8 +212,10 @@ class ToolApiProviderListToolsApi(Resource):
     @login_required
     @account_initialization_required
     def get(self):
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user = current_user
+
+        user_id = user.id
+        tenant_id = user.current_tenant_id
 
         parser = reqparse.RequestParser()
 
@@ -205,11 +237,13 @@ class ToolApiProviderUpdateApi(Resource):
     @login_required
     @account_initialization_required
     def post(self):
-        if not current_user.is_admin_or_owner:
+        user = current_user
+
+        if not user.is_admin_or_owner:
             raise Forbidden()
 
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user_id = user.id
+        tenant_id = user.current_tenant_id
 
         parser = reqparse.RequestParser()
         parser.add_argument("credentials", type=dict, required=True, nullable=False, location="json")
@@ -244,11 +278,13 @@ class ToolApiProviderDeleteApi(Resource):
     @login_required
     @account_initialization_required
     def post(self):
-        if not current_user.is_admin_or_owner:
+        user = current_user
+
+        if not user.is_admin_or_owner:
             raise Forbidden()
 
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user_id = user.id
+        tenant_id = user.current_tenant_id
 
         parser = reqparse.RequestParser()
 
@@ -268,8 +304,10 @@ class ToolApiProviderGetApi(Resource):
     @login_required
     @account_initialization_required
     def get(self):
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user = current_user
+
+        user_id = user.id
+        tenant_id = user.current_tenant_id
 
         parser = reqparse.RequestParser()
 
@@ -289,7 +327,11 @@ class ToolBuiltinProviderCredentialsSchemaApi(Resource):
     @login_required
     @account_initialization_required
     def get(self, provider):
-        return BuiltinToolManageService.list_builtin_provider_credentials_schema(provider)
+        user = current_user
+
+        tenant_id = user.current_tenant_id
+
+        return BuiltinToolManageService.list_builtin_provider_credentials_schema(provider, tenant_id)
 
 
 class ToolApiProviderSchemaApi(Resource):
@@ -340,11 +382,13 @@ class ToolWorkflowProviderCreateApi(Resource):
     @login_required
     @account_initialization_required
     def post(self):
-        if not current_user.is_admin_or_owner:
+        user = current_user
+
+        if not user.is_admin_or_owner:
             raise Forbidden()
 
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user_id = user.id
+        tenant_id = user.current_tenant_id
 
         reqparser = reqparse.RequestParser()
         reqparser.add_argument("workflow_app_id", type=uuid_value, required=True, nullable=False, location="json")
@@ -377,11 +421,13 @@ class ToolWorkflowProviderUpdateApi(Resource):
     @login_required
     @account_initialization_required
     def post(self):
-        if not current_user.is_admin_or_owner:
+        user = current_user
+
+        if not user.is_admin_or_owner:
             raise Forbidden()
 
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user_id = user.id
+        tenant_id = user.current_tenant_id
 
         reqparser = reqparse.RequestParser()
         reqparser.add_argument("workflow_tool_id", type=uuid_value, required=True, nullable=False, location="json")
@@ -417,11 +463,13 @@ class ToolWorkflowProviderDeleteApi(Resource):
     @login_required
     @account_initialization_required
     def post(self):
-        if not current_user.is_admin_or_owner:
+        user = current_user
+
+        if not user.is_admin_or_owner:
             raise Forbidden()
 
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user_id = user.id
+        tenant_id = user.current_tenant_id
 
         reqparser = reqparse.RequestParser()
         reqparser.add_argument("workflow_tool_id", type=uuid_value, required=True, nullable=False, location="json")
@@ -440,8 +488,10 @@ class ToolWorkflowProviderGetApi(Resource):
     @login_required
     @account_initialization_required
     def get(self):
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user = current_user
+
+        user_id = user.id
+        tenant_id = user.current_tenant_id
 
         parser = reqparse.RequestParser()
         parser.add_argument("workflow_tool_id", type=uuid_value, required=False, nullable=True, location="args")
@@ -472,8 +522,10 @@ class ToolWorkflowProviderListToolApi(Resource):
     @login_required
     @account_initialization_required
     def get(self):
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user = current_user
+
+        user_id = user.id
+        tenant_id = user.current_tenant_id
 
         parser = reqparse.RequestParser()
         parser.add_argument("workflow_tool_id", type=uuid_value, required=True, nullable=False, location="args")
@@ -494,8 +546,10 @@ class ToolBuiltinListApi(Resource):
     @login_required
     @account_initialization_required
     def get(self):
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user = current_user
+
+        user_id = user.id
+        tenant_id = user.current_tenant_id
 
         return jsonable_encoder(
             [
@@ -513,8 +567,10 @@ class ToolApiListApi(Resource):
     @login_required
     @account_initialization_required
     def get(self):
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user = current_user
+
+        user_id = user.id
+        tenant_id = user.current_tenant_id
 
         return jsonable_encoder(
             [
@@ -532,8 +588,10 @@ class ToolWorkflowListApi(Resource):
     @login_required
     @account_initialization_required
     def get(self):
-        user_id = current_user.id
-        tenant_id = current_user.current_tenant_id
+        user = current_user
+
+        user_id = user.id
+        tenant_id = user.current_tenant_id
 
         return jsonable_encoder(
             [
@@ -559,16 +617,18 @@ class ToolLabelsApi(Resource):
 api.add_resource(ToolProviderListApi, "/workspaces/current/tool-providers")
 
 # builtin tool provider
-api.add_resource(ToolBuiltinProviderListToolsApi, "/workspaces/current/tool-provider/builtin/<provider>/tools")
-api.add_resource(ToolBuiltinProviderDeleteApi, "/workspaces/current/tool-provider/builtin/<provider>/delete")
-api.add_resource(ToolBuiltinProviderUpdateApi, "/workspaces/current/tool-provider/builtin/<provider>/update")
+api.add_resource(ToolBuiltinProviderListToolsApi, "/workspaces/current/tool-provider/builtin/<path:provider>/tools")
+api.add_resource(ToolBuiltinProviderInfoApi, "/workspaces/current/tool-provider/builtin/<path:provider>/info")
+api.add_resource(ToolBuiltinProviderDeleteApi, "/workspaces/current/tool-provider/builtin/<path:provider>/delete")
+api.add_resource(ToolBuiltinProviderUpdateApi, "/workspaces/current/tool-provider/builtin/<path:provider>/update")
 api.add_resource(
-    ToolBuiltinProviderGetCredentialsApi, "/workspaces/current/tool-provider/builtin/<provider>/credentials"
+    ToolBuiltinProviderGetCredentialsApi, "/workspaces/current/tool-provider/builtin/<path:provider>/credentials"
 )
 api.add_resource(
-    ToolBuiltinProviderCredentialsSchemaApi, "/workspaces/current/tool-provider/builtin/<provider>/credentials_schema"
+    ToolBuiltinProviderCredentialsSchemaApi,
+    "/workspaces/current/tool-provider/builtin/<path:provider>/credentials_schema",
 )
-api.add_resource(ToolBuiltinProviderIconApi, "/workspaces/current/tool-provider/builtin/<provider>/icon")
+api.add_resource(ToolBuiltinProviderIconApi, "/workspaces/current/tool-provider/builtin/<path:provider>/icon")
 
 # api tool provider
 api.add_resource(ToolApiProviderAddApi, "/workspaces/current/tool-provider/api/add")

api/controllers/console/workspace/workspace.py

@@ -1,8 +1,8 @@
 import logging
 
 from flask import request
-from flask_login import current_user
-from flask_restful import Resource, fields, inputs, marshal, marshal_with, reqparse
+from flask_login import current_user  # type: ignore
+from flask_restful import Resource, fields, inputs, marshal, marshal_with, reqparse  # type: ignore
 from werkzeug.exceptions import Unauthorized
 
 import services
@@ -82,11 +82,7 @@ class WorkspaceListApi(Resource):
         parser.add_argument("limit", type=inputs.int_range(1, 100), required=False, default=20, location="args")
         args = parser.parse_args()
 
-        tenants = (
-            db.session.query(Tenant)
-            .order_by(Tenant.created_at.desc())
-            .paginate(page=args["page"], per_page=args["limit"])
-        )
+        tenants = Tenant.query.order_by(Tenant.created_at.desc()).paginate(page=args["page"], per_page=args["limit"])
 
         has_more = False
         if len(tenants.items) == args["limit"]:
@@ -151,6 +147,8 @@ class SwitchWorkspaceApi(Resource):
             raise AccountNotLinkTenantError("Account not link tenant")
 
         new_tenant = db.session.query(Tenant).get(args["tenant_id"])  # Get new tenant
+        if new_tenant is None:
+            raise ValueError("Tenant not found")
 
         return {"result": "success", "new_tenant": marshal(WorkspaceService.get_tenant_info(new_tenant), tenant_fields)}
 
@@ -166,7 +164,7 @@ class CustomConfigWorkspaceApi(Resource):
         parser.add_argument("replace_webapp_logo", type=str, location="json")
         args = parser.parse_args()
 
-        tenant = db.session.query(Tenant).filter(Tenant.id == current_user.current_tenant_id).one_or_404()
+        tenant = Tenant.query.filter(Tenant.id == current_user.current_tenant_id).one_or_404()
 
         custom_config_dict = {
             "remove_webapp_brand": args["remove_webapp_brand"],

api/controllers/console/wraps.py

@@ -3,10 +3,11 @@ import os
 from functools import wraps
 
 from flask import abort, request
-from flask_login import current_user
+from flask_login import current_user  # type: ignore
 
 from configs import dify_config
 from controllers.console.workspace.error import AccountNotInitializedError
+from extensions.ext_database import db
 from models.model import DifySetup
 from services.feature_service import FeatureService, LicenseStatus
 from services.operation_service import OperationService
@@ -121,8 +122,8 @@ def cloud_utm_record(view):
                 utm_info = request.cookies.get("utm_info")
 
                 if utm_info:
-                    utm_info = json.loads(utm_info)
-                    OperationService.record_utm(current_user.current_tenant_id, utm_info)
+                    utm_info_dict: dict = json.loads(utm_info)
+                    OperationService.record_utm(current_user.current_tenant_id, utm_info_dict)
         except Exception as e:
             pass
         return view(*args, **kwargs)
@@ -134,9 +135,13 @@ def setup_required(view):
     @wraps(view)
     def decorated(*args, **kwargs):
         # check setup
-        if dify_config.EDITION == "SELF_HOSTED" and os.environ.get("INIT_PASSWORD") and not DifySetup.query.first():
+        if (
+            dify_config.EDITION == "SELF_HOSTED"
+            and os.environ.get("INIT_PASSWORD")
+            and not db.session.query(DifySetup).first()
+        ):
             raise NotInitValidateError()
-        elif dify_config.EDITION == "SELF_HOSTED" and not DifySetup.query.first():
+        elif dify_config.EDITION == "SELF_HOSTED" and not db.session.query(DifySetup).first():
             raise NotSetupError()
 
         return view(*args, **kwargs)

api/controllers/files/__init__.py

@@ -6,4 +6,4 @@ bp = Blueprint("files", __name__)
 api = ExternalApi(bp)
 
 
-from . import image_preview, tool_files
+from . import image_preview, tool_files, upload

api/controllers/files/image_preview.py

@@ -1,5 +1,5 @@
 from flask import Response, request
-from flask_restful import Resource, reqparse
+from flask_restful import Resource, reqparse  # type: ignore
 from werkzeug.exceptions import NotFound
 
 import services

api/controllers/files/tool_files.py

@@ -1,5 +1,5 @@
 from flask import Response
-from flask_restful import Resource, reqparse
+from flask_restful import Resource, reqparse  # type: ignore
 from werkzeug.exceptions import Forbidden, NotFound
 
 from controllers.files import api

api/controllers/files/upload.py

@@ -0,0 +1,69 @@
+from flask import request
+from flask_restful import Resource, marshal_with  # type: ignore
+from werkzeug.exceptions import Forbidden
+
+import services
+from controllers.console.wraps import setup_required
+from controllers.files import api
+from controllers.files.error import UnsupportedFileTypeError
+from controllers.inner_api.plugin.wraps import get_user
+from controllers.service_api.app.error import FileTooLargeError
+from core.file.helpers import verify_plugin_file_signature
+from fields.file_fields import file_fields
+from services.file_service import FileService
+
+
+class PluginUploadFileApi(Resource):
+    @setup_required
+    @marshal_with(file_fields)
+    def post(self):
+        # get file from request
+        file = request.files["file"]
+
+        timestamp = request.args.get("timestamp")
+        nonce = request.args.get("nonce")
+        sign = request.args.get("sign")
+        tenant_id = request.args.get("tenant_id")
+        if not tenant_id:
+            raise Forbidden("Invalid request.")
+
+        user_id = request.args.get("user_id")
+        user = get_user(tenant_id, user_id)
+
+        filename = file.filename
+        mimetype = file.mimetype
+
+        if not filename or not mimetype:
+            raise Forbidden("Invalid request.")
+
+        if not timestamp or not nonce or not sign:
+            raise Forbidden("Invalid request.")
+
+        if not verify_plugin_file_signature(
+            filename=filename,
+            mimetype=mimetype,
+            tenant_id=tenant_id,
+            user_id=user_id,
+            timestamp=timestamp,
+            nonce=nonce,
+            sign=sign,
+        ):
+            raise Forbidden("Invalid request.")
+
+        try:
+            upload_file = FileService.upload_file(
+                filename=filename,
+                content=file.read(),
+                mimetype=mimetype,
+                user=user,
+                source=None,
+            )
+        except services.errors.file.FileTooLargeError as file_too_large_error:
+            raise FileTooLargeError(file_too_large_error.description)
+        except services.errors.file.UnsupportedFileTypeError:
+            raise UnsupportedFileTypeError()
+
+        return upload_file, 201
+
+
+api.add_resource(PluginUploadFileApi, "/files/upload/for-plugin")

api/controllers/inner_api/__init__.py

@@ -5,4 +5,5 @@ from libs.external_api import ExternalApi
 bp = Blueprint("inner_api", __name__, url_prefix="/inner/api")
 api = ExternalApi(bp)
 
+from .plugin import plugin
 from .workspace import workspace

api/controllers/inner_api/plugin/plugin.py

@@ -0,0 +1,293 @@
+from flask_restful import Resource  # type: ignore
+
+from controllers.console.wraps import setup_required
+from controllers.inner_api import api
+from controllers.inner_api.plugin.wraps import get_user_tenant, plugin_data
+from controllers.inner_api.wraps import plugin_inner_api_only
+from core.file.helpers import get_signed_file_url_for_plugin
+from core.model_runtime.utils.encoders import jsonable_encoder
+from core.plugin.backwards_invocation.app import PluginAppBackwardsInvocation
+from core.plugin.backwards_invocation.base import BaseBackwardsInvocationResponse
+from core.plugin.backwards_invocation.encrypt import PluginEncrypter
+from core.plugin.backwards_invocation.model import PluginModelBackwardsInvocation
+from core.plugin.backwards_invocation.node import PluginNodeBackwardsInvocation
+from core.plugin.backwards_invocation.tool import PluginToolBackwardsInvocation
+from core.plugin.entities.request import (
+    RequestInvokeApp,
+    RequestInvokeEncrypt,
+    RequestInvokeLLM,
+    RequestInvokeModeration,
+    RequestInvokeParameterExtractorNode,
+    RequestInvokeQuestionClassifierNode,
+    RequestInvokeRerank,
+    RequestInvokeSpeech2Text,
+    RequestInvokeSummary,
+    RequestInvokeTextEmbedding,
+    RequestInvokeTool,
+    RequestInvokeTTS,
+    RequestRequestUploadFile,
+)
+from core.tools.entities.tool_entities import ToolProviderType
+from libs.helper import compact_generate_response
+from models.account import Account, Tenant
+from models.model import EndUser
+
+
+class PluginInvokeLLMApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @get_user_tenant
+    @plugin_data(payload_type=RequestInvokeLLM)
+    def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeLLM):
+        def generator():
+            response = PluginModelBackwardsInvocation.invoke_llm(user_model.id, tenant_model, payload)
+            return PluginModelBackwardsInvocation.convert_to_event_stream(response)
+
+        return compact_generate_response(generator())
+
+
+class PluginInvokeTextEmbeddingApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @get_user_tenant
+    @plugin_data(payload_type=RequestInvokeTextEmbedding)
+    def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeTextEmbedding):
+        try:
+            return jsonable_encoder(
+                BaseBackwardsInvocationResponse(
+                    data=PluginModelBackwardsInvocation.invoke_text_embedding(
+                        user_id=user_model.id,
+                        tenant=tenant_model,
+                        payload=payload,
+                    )
+                )
+            )
+        except Exception as e:
+            return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
+
+
+class PluginInvokeRerankApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @get_user_tenant
+    @plugin_data(payload_type=RequestInvokeRerank)
+    def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeRerank):
+        try:
+            return jsonable_encoder(
+                BaseBackwardsInvocationResponse(
+                    data=PluginModelBackwardsInvocation.invoke_rerank(
+                        user_id=user_model.id,
+                        tenant=tenant_model,
+                        payload=payload,
+                    )
+                )
+            )
+        except Exception as e:
+            return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
+
+
+class PluginInvokeTTSApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @get_user_tenant
+    @plugin_data(payload_type=RequestInvokeTTS)
+    def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeTTS):
+        def generator():
+            response = PluginModelBackwardsInvocation.invoke_tts(
+                user_id=user_model.id,
+                tenant=tenant_model,
+                payload=payload,
+            )
+            return PluginModelBackwardsInvocation.convert_to_event_stream(response)
+
+        return compact_generate_response(generator())
+
+
+class PluginInvokeSpeech2TextApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @get_user_tenant
+    @plugin_data(payload_type=RequestInvokeSpeech2Text)
+    def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeSpeech2Text):
+        try:
+            return jsonable_encoder(
+                BaseBackwardsInvocationResponse(
+                    data=PluginModelBackwardsInvocation.invoke_speech2text(
+                        user_id=user_model.id,
+                        tenant=tenant_model,
+                        payload=payload,
+                    )
+                )
+            )
+        except Exception as e:
+            return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
+
+
+class PluginInvokeModerationApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @get_user_tenant
+    @plugin_data(payload_type=RequestInvokeModeration)
+    def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeModeration):
+        try:
+            return jsonable_encoder(
+                BaseBackwardsInvocationResponse(
+                    data=PluginModelBackwardsInvocation.invoke_moderation(
+                        user_id=user_model.id,
+                        tenant=tenant_model,
+                        payload=payload,
+                    )
+                )
+            )
+        except Exception as e:
+            return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
+
+
+class PluginInvokeToolApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @get_user_tenant
+    @plugin_data(payload_type=RequestInvokeTool)
+    def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeTool):
+        def generator():
+            return PluginToolBackwardsInvocation.convert_to_event_stream(
+                PluginToolBackwardsInvocation.invoke_tool(
+                    tenant_id=tenant_model.id,
+                    user_id=user_model.id,
+                    tool_type=ToolProviderType.value_of(payload.tool_type),
+                    provider=payload.provider,
+                    tool_name=payload.tool,
+                    tool_parameters=payload.tool_parameters,
+                ),
+            )
+
+        return compact_generate_response(generator())
+
+
+class PluginInvokeParameterExtractorNodeApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @get_user_tenant
+    @plugin_data(payload_type=RequestInvokeParameterExtractorNode)
+    def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeParameterExtractorNode):
+        try:
+            return jsonable_encoder(
+                BaseBackwardsInvocationResponse(
+                    data=PluginNodeBackwardsInvocation.invoke_parameter_extractor(
+                        tenant_id=tenant_model.id,
+                        user_id=user_model.id,
+                        parameters=payload.parameters,
+                        model_config=payload.model,
+                        instruction=payload.instruction,
+                        query=payload.query,
+                    )
+                )
+            )
+        except Exception as e:
+            return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
+
+
+class PluginInvokeQuestionClassifierNodeApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @get_user_tenant
+    @plugin_data(payload_type=RequestInvokeQuestionClassifierNode)
+    def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeQuestionClassifierNode):
+        try:
+            return jsonable_encoder(
+                BaseBackwardsInvocationResponse(
+                    data=PluginNodeBackwardsInvocation.invoke_question_classifier(
+                        tenant_id=tenant_model.id,
+                        user_id=user_model.id,
+                        query=payload.query,
+                        model_config=payload.model,
+                        classes=payload.classes,
+                        instruction=payload.instruction,
+                    )
+                )
+            )
+        except Exception as e:
+            return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
+
+
+class PluginInvokeAppApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @get_user_tenant
+    @plugin_data(payload_type=RequestInvokeApp)
+    def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeApp):
+        response = PluginAppBackwardsInvocation.invoke_app(
+            app_id=payload.app_id,
+            user_id=user_model.id,
+            tenant_id=tenant_model.id,
+            conversation_id=payload.conversation_id,
+            query=payload.query,
+            stream=payload.response_mode == "streaming",
+            inputs=payload.inputs,
+            files=payload.files,
+        )
+
+        return compact_generate_response(PluginAppBackwardsInvocation.convert_to_event_stream(response))
+
+
+class PluginInvokeEncryptApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @get_user_tenant
+    @plugin_data(payload_type=RequestInvokeEncrypt)
+    def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeEncrypt):
+        """
+        encrypt or decrypt data
+        """
+        try:
+            return BaseBackwardsInvocationResponse(
+                data=PluginEncrypter.invoke_encrypt(tenant_model, payload)
+            ).model_dump()
+        except Exception as e:
+            return BaseBackwardsInvocationResponse(error=str(e)).model_dump()
+
+
+class PluginInvokeSummaryApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @get_user_tenant
+    @plugin_data(payload_type=RequestInvokeSummary)
+    def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeSummary):
+        try:
+            return BaseBackwardsInvocationResponse(
+                data={
+                    "summary": PluginModelBackwardsInvocation.invoke_summary(
+                        user_id=user_model.id,
+                        tenant=tenant_model,
+                        payload=payload,
+                    )
+                }
+            ).model_dump()
+        except Exception as e:
+            return BaseBackwardsInvocationResponse(error=str(e)).model_dump()
+
+
+class PluginUploadFileRequestApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @get_user_tenant
+    @plugin_data(payload_type=RequestRequestUploadFile)
+    def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestRequestUploadFile):
+        # generate signed url
+        url = get_signed_file_url_for_plugin(payload.filename, payload.mimetype, tenant_model.id, user_model.id)
+        return BaseBackwardsInvocationResponse(data={"url": url}).model_dump()
+
+
+api.add_resource(PluginInvokeLLMApi, "/invoke/llm")
+api.add_resource(PluginInvokeTextEmbeddingApi, "/invoke/text-embedding")
+api.add_resource(PluginInvokeRerankApi, "/invoke/rerank")
+api.add_resource(PluginInvokeTTSApi, "/invoke/tts")
+api.add_resource(PluginInvokeSpeech2TextApi, "/invoke/speech2text")
+api.add_resource(PluginInvokeModerationApi, "/invoke/moderation")
+api.add_resource(PluginInvokeToolApi, "/invoke/tool")
+api.add_resource(PluginInvokeParameterExtractorNodeApi, "/invoke/parameter-extractor")
+api.add_resource(PluginInvokeQuestionClassifierNodeApi, "/invoke/question-classifier")
+api.add_resource(PluginInvokeAppApi, "/invoke/app")
+api.add_resource(PluginInvokeEncryptApi, "/invoke/encrypt")
+api.add_resource(PluginInvokeSummaryApi, "/invoke/summary")
+api.add_resource(PluginUploadFileRequestApi, "/upload/file/request")