mirror of
https://github.com/langgenius/dify.git
synced 2025-12-21 15:02:26 +00:00
Compare commits
3 Commits
feat/group
...
feat/add-k
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cf08295dde | ||
|
|
d0e9fccc9d | ||
|
|
4cebaa331e |
@@ -18,3 +18,18 @@ class EnterpriseFeatureConfig(BaseSettings):
|
|||||||
description="Allow customization of the enterprise logo.",
|
description="Allow customization of the enterprise logo.",
|
||||||
default=False,
|
default=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
UPLOAD_KNOWLEDGE_PIPELINE_TEMPLATE_TOKEN: str = Field(
|
||||||
|
description="Token for uploading knowledge pipeline template.",
|
||||||
|
default="",
|
||||||
|
)
|
||||||
|
|
||||||
|
KNOWLEDGE_PIPELINE_TEMPLATE_COPYRIGHT: str = Field(
|
||||||
|
description="Knowledge pipeline template copyright.",
|
||||||
|
default="Copyright 2023 Dify",
|
||||||
|
)
|
||||||
|
|
||||||
|
KNOWLEDGE_PIPELINE_TEMPLATE_PRIVACY_POLICY: str = Field(
|
||||||
|
description="Knowledge pipeline template privacy policy.",
|
||||||
|
default="https://dify.ai",
|
||||||
|
)
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from flask import request
|
from flask import request
|
||||||
from flask_restx import Resource, reqparse
|
from flask_restx import Resource, reqparse
|
||||||
@@ -14,24 +16,17 @@ from controllers.console.wraps import (
|
|||||||
from extensions.ext_database import db
|
from extensions.ext_database import db
|
||||||
from libs.login import login_required
|
from libs.login import login_required
|
||||||
from models.dataset import PipelineCustomizedTemplate
|
from models.dataset import PipelineCustomizedTemplate
|
||||||
from services.entities.knowledge_entities.rag_pipeline_entities import PipelineTemplateInfoEntity
|
from services.entities.knowledge_entities.rag_pipeline_entities import (
|
||||||
|
PipelineBuiltInTemplateEntity,
|
||||||
|
PipelineBuiltInTemplateInstallEntity,
|
||||||
|
PipelineBuiltInTemplateUpdateEntity,
|
||||||
|
PipelineTemplateInfoEntity,
|
||||||
|
)
|
||||||
from services.rag_pipeline.rag_pipeline import RagPipelineService
|
from services.rag_pipeline.rag_pipeline import RagPipelineService
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def _validate_name(name):
|
|
||||||
if not name or len(name) < 1 or len(name) > 40:
|
|
||||||
raise ValueError("Name must be between 1 to 40 characters.")
|
|
||||||
return name
|
|
||||||
|
|
||||||
|
|
||||||
def _validate_description_length(description):
|
|
||||||
if len(description) > 400:
|
|
||||||
raise ValueError("Description cannot exceed 400 characters.")
|
|
||||||
return description
|
|
||||||
|
|
||||||
|
|
||||||
class PipelineTemplateListApi(Resource):
|
class PipelineTemplateListApi(Resource):
|
||||||
@setup_required
|
@setup_required
|
||||||
@login_required
|
@login_required
|
||||||
@@ -146,6 +141,302 @@ class PublishCustomizedPipelineTemplateApi(Resource):
|
|||||||
return {"result": "success"}
|
return {"result": "success"}
|
||||||
|
|
||||||
|
|
||||||
|
class PipelineTemplateInstallApi(Resource):
|
||||||
|
"""API endpoint for installing built-in pipeline templates"""
|
||||||
|
|
||||||
|
def post(self):
|
||||||
|
"""
|
||||||
|
Install a built-in pipeline template
|
||||||
|
|
||||||
|
Args:
|
||||||
|
template_id: The template ID from URL parameter
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Success response or error with appropriate HTTP status
|
||||||
|
"""
|
||||||
|
parser = reqparse.RequestParser()
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"language",
|
||||||
|
type=str,
|
||||||
|
location="form",
|
||||||
|
required=True,
|
||||||
|
default="en-US",
|
||||||
|
choices=["en-US", "zh-CN", "ja-JP"],
|
||||||
|
help="Template language code"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"name",
|
||||||
|
type=str,
|
||||||
|
location="form",
|
||||||
|
required=True,
|
||||||
|
default="New Pipeline Template",
|
||||||
|
help="Template name (1-200 characters)"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"description",
|
||||||
|
type=str,
|
||||||
|
location="form",
|
||||||
|
required=False,
|
||||||
|
default="",
|
||||||
|
help="Template description (max 1000 characters)"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"position",
|
||||||
|
type=int,
|
||||||
|
location="form",
|
||||||
|
required=False,
|
||||||
|
default=1,
|
||||||
|
help="Template position"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"icon",
|
||||||
|
type=str,
|
||||||
|
location="form",
|
||||||
|
required=False,
|
||||||
|
help="Template icon"
|
||||||
|
)
|
||||||
|
|
||||||
|
template_args = parser.parse_args()
|
||||||
|
|
||||||
|
# Additional validation
|
||||||
|
if template_args.get("name"):
|
||||||
|
template_args["name"] = _validate_name(template_args["name"])
|
||||||
|
|
||||||
|
if template_args.get("icon"):
|
||||||
|
template_args["icon"] = json.loads(template_args["icon"])
|
||||||
|
|
||||||
|
if template_args.get("description") and len(template_args["description"]) > 1000:
|
||||||
|
raise ValueError("Description must not exceed 1000 characters")
|
||||||
|
try:
|
||||||
|
# Extract and validate Bearer token
|
||||||
|
auth_token = _extract_bearer_token()
|
||||||
|
|
||||||
|
# Process uploaded template file
|
||||||
|
file_content = _process_template_file()
|
||||||
|
template_args["yaml_content"] = file_content
|
||||||
|
|
||||||
|
# Create template entity
|
||||||
|
pipeline_built_in_template_entity = PipelineBuiltInTemplateInstallEntity(**template_args)
|
||||||
|
|
||||||
|
# Install the template
|
||||||
|
rag_pipeline_service = RagPipelineService()
|
||||||
|
rag_pipeline_service.insert_built_in_pipeline_template(
|
||||||
|
pipeline_built_in_template_entity, auth_token
|
||||||
|
)
|
||||||
|
|
||||||
|
return {"result": "success", "message": "Template installed successfully"}, 200
|
||||||
|
|
||||||
|
except ValueError as e:
|
||||||
|
logger.exception("Validation error in template installation")
|
||||||
|
return {"error": str(e)}, 400
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("Unexpected error in template installation")
|
||||||
|
return {"error": "An unexpected error occurred during template installation"}, 500
|
||||||
|
|
||||||
|
|
||||||
|
class PipelineTemplateUnpdateApi(Resource):
|
||||||
|
"""API endpoint for updating built-in pipeline templates"""
|
||||||
|
def patch(self, template_id: str):
|
||||||
|
"""
|
||||||
|
Update a built-in pipeline template
|
||||||
|
|
||||||
|
Args:
|
||||||
|
template_id: The template ID to update (from URL parameter)
|
||||||
|
"""
|
||||||
|
parser = reqparse.RequestParser()
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"language",
|
||||||
|
type=str,
|
||||||
|
location="form",
|
||||||
|
required=True,
|
||||||
|
default="en-US",
|
||||||
|
choices=["en-US", "zh-CN", "ja-JP"],
|
||||||
|
help="Template language code"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"name",
|
||||||
|
type=str,
|
||||||
|
location="form",
|
||||||
|
required=True,
|
||||||
|
default="New Pipeline Template",
|
||||||
|
help="Template name (1-200 characters)"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"description",
|
||||||
|
type=str,
|
||||||
|
location="form",
|
||||||
|
required=False,
|
||||||
|
default="",
|
||||||
|
help="Template description (max 1000 characters)"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"position",
|
||||||
|
type=int,
|
||||||
|
location="form",
|
||||||
|
required=False,
|
||||||
|
default=1,
|
||||||
|
help="Template position"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"icon",
|
||||||
|
type=str,
|
||||||
|
location="form",
|
||||||
|
required=False,
|
||||||
|
help="Template icon"
|
||||||
|
)
|
||||||
|
|
||||||
|
template_args = parser.parse_args()
|
||||||
|
try:
|
||||||
|
# Extract and validate Bearer token
|
||||||
|
auth_token = _extract_bearer_token()
|
||||||
|
|
||||||
|
if template_args.get("icon"):
|
||||||
|
template_args["icon"] = json.loads(template_args["icon"])
|
||||||
|
|
||||||
|
if "file" in request.files:
|
||||||
|
file_content = request.files["file"].read().decode("utf-8")
|
||||||
|
template_args["yaml_content"] = file_content
|
||||||
|
else:
|
||||||
|
template_args["yaml_content"] = None
|
||||||
|
|
||||||
|
# Validate template_id
|
||||||
|
if not template_id or not template_id.strip():
|
||||||
|
raise ValueError("Template ID is required")
|
||||||
|
|
||||||
|
template_args["template_id"] = template_id
|
||||||
|
|
||||||
|
pipeline_built_in_template_entity = PipelineBuiltInTemplateUpdateEntity(**template_args)
|
||||||
|
|
||||||
|
# Update the template
|
||||||
|
rag_pipeline_service = RagPipelineService()
|
||||||
|
rag_pipeline_service.update_built_in_pipeline_template(pipeline_built_in_template_entity, auth_token)
|
||||||
|
|
||||||
|
return {"result": "success", "message": "Template updated successfully"}, 200
|
||||||
|
|
||||||
|
except ValueError as e:
|
||||||
|
logger.exception("Validation error in template update")
|
||||||
|
return {"error": str(e)}, 400
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("Unexpected error in template update")
|
||||||
|
return {"error": "An unexpected error occurred during template update"}, 500
|
||||||
|
|
||||||
|
def delete(self, template_id: str):
|
||||||
|
"""
|
||||||
|
Uninstall a built-in pipeline template
|
||||||
|
|
||||||
|
Args:
|
||||||
|
template_id: The template ID to uninstall (from URL parameter)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Success response or error with appropriate HTTP status
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# Extract and validate Bearer token
|
||||||
|
auth_token = _extract_bearer_token()
|
||||||
|
|
||||||
|
# Validate template_id
|
||||||
|
if not template_id or not template_id.strip():
|
||||||
|
raise ValueError("Template ID is required")
|
||||||
|
|
||||||
|
# Uninstall the template
|
||||||
|
rag_pipeline_service = RagPipelineService()
|
||||||
|
rag_pipeline_service.uninstall_built_in_pipeline_template(
|
||||||
|
template_id.strip(), auth_token
|
||||||
|
)
|
||||||
|
|
||||||
|
return {"result": "success", "message": "Template uninstalled successfully"}, 200
|
||||||
|
|
||||||
|
except ValueError as e:
|
||||||
|
logger.exception("Validation error in template uninstallation")
|
||||||
|
return {"error": str(e)}, 400
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("Unexpected error in template uninstallation")
|
||||||
|
return {"error": "An unexpected error occurred during template uninstallation"}, 500
|
||||||
|
|
||||||
|
def _extract_bearer_token() -> str:
|
||||||
|
"""
|
||||||
|
Extract and validate Bearer token from Authorization header
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The extracted token string
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: If token is missing or invalid
|
||||||
|
"""
|
||||||
|
auth_header = request.headers.get("Authorization", "").strip()
|
||||||
|
|
||||||
|
if not auth_header:
|
||||||
|
raise ValueError("Authorization header is required")
|
||||||
|
|
||||||
|
if not auth_header.startswith("Bearer "):
|
||||||
|
raise ValueError("Authorization header must start with 'Bearer '")
|
||||||
|
|
||||||
|
token_parts = auth_header.split(" ", 1)
|
||||||
|
if len(token_parts) != 2:
|
||||||
|
raise ValueError("Invalid Authorization header format")
|
||||||
|
|
||||||
|
auth_token = token_parts[1].strip()
|
||||||
|
if not auth_token:
|
||||||
|
raise ValueError("Bearer token cannot be empty")
|
||||||
|
|
||||||
|
return auth_token
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_name(name: str) -> str:
|
||||||
|
"""
|
||||||
|
Validate template name
|
||||||
|
|
||||||
|
Args:
|
||||||
|
name: Template name to validate
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Validated and trimmed name
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: If name is invalid
|
||||||
|
"""
|
||||||
|
name = name.strip()
|
||||||
|
if not name or len(name) < 1 or len(name) > 200:
|
||||||
|
raise ValueError("Template name must be between 1 and 200 characters")
|
||||||
|
return name
|
||||||
|
|
||||||
|
def _process_template_file() -> str:
|
||||||
|
"""
|
||||||
|
Process and validate uploaded template file
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
File content as string
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: If file is missing or invalid
|
||||||
|
"""
|
||||||
|
if "file" not in request.files:
|
||||||
|
raise ValueError("Template file is required")
|
||||||
|
|
||||||
|
file = request.files["file"]
|
||||||
|
|
||||||
|
# Validate file
|
||||||
|
if not file or not file.filename:
|
||||||
|
raise ValueError("No file selected")
|
||||||
|
|
||||||
|
filename = file.filename.strip()
|
||||||
|
if not filename:
|
||||||
|
raise ValueError("File name cannot be empty")
|
||||||
|
|
||||||
|
# Check file extension
|
||||||
|
if not filename.lower().endswith(".pipeline"):
|
||||||
|
raise ValueError("Template file must be a pipeline file (.pipeline)")
|
||||||
|
|
||||||
|
try:
|
||||||
|
file_content = file.read().decode("utf-8")
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
raise ValueError("Template file must be valid UTF-8 text")
|
||||||
|
|
||||||
|
return file_content
|
||||||
|
|
||||||
|
|
||||||
api.add_resource(
|
api.add_resource(
|
||||||
PipelineTemplateListApi,
|
PipelineTemplateListApi,
|
||||||
"/rag/pipeline/templates",
|
"/rag/pipeline/templates",
|
||||||
@@ -162,3 +453,11 @@ api.add_resource(
|
|||||||
PublishCustomizedPipelineTemplateApi,
|
PublishCustomizedPipelineTemplateApi,
|
||||||
"/rag/pipelines/<string:pipeline_id>/customized/publish",
|
"/rag/pipelines/<string:pipeline_id>/customized/publish",
|
||||||
)
|
)
|
||||||
|
api.add_resource(
|
||||||
|
PipelineTemplateInstallApi,
|
||||||
|
"/rag/pipeline/built-in/templates/install",
|
||||||
|
)
|
||||||
|
api.add_resource(
|
||||||
|
PipelineTemplateUninstallApi,
|
||||||
|
"/rag/pipeline/built-in/templates/<string:template_id>/uninstall",
|
||||||
|
)
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
"""remove-builtin-template-user
|
||||||
|
|
||||||
|
Revision ID: bf0bcbf45396
|
||||||
|
Revises: 68519ad5cd18
|
||||||
|
Create Date: 2025-09-25 16:50:32.245503
|
||||||
|
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import models as models
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = 'bf0bcbf45396'
|
||||||
|
down_revision = '68519ad5cd18'
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
|
||||||
|
with op.batch_alter_table('pipeline_built_in_templates', schema=None) as batch_op:
|
||||||
|
batch_op.drop_column('updated_by')
|
||||||
|
batch_op.drop_column('created_by')
|
||||||
|
|
||||||
|
# ### end Alembic commands ###
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
|
||||||
|
with op.batch_alter_table('pipeline_built_in_templates', schema=None) as batch_op:
|
||||||
|
batch_op.add_column(sa.Column('created_by', sa.UUID(), autoincrement=False, nullable=False))
|
||||||
|
batch_op.add_column(sa.Column('updated_by', sa.UUID(), autoincrement=False, nullable=True))
|
||||||
|
|
||||||
|
# ### end Alembic commands ###
|
||||||
@@ -1239,15 +1239,6 @@ class PipelineBuiltInTemplate(Base): # type: ignore[name-defined]
|
|||||||
language = db.Column(db.String(255), nullable=False)
|
language = db.Column(db.String(255), nullable=False)
|
||||||
created_at = db.Column(db.DateTime, nullable=False, server_default=func.current_timestamp())
|
created_at = db.Column(db.DateTime, nullable=False, server_default=func.current_timestamp())
|
||||||
updated_at = db.Column(db.DateTime, nullable=False, server_default=func.current_timestamp())
|
updated_at = db.Column(db.DateTime, nullable=False, server_default=func.current_timestamp())
|
||||||
created_by = db.Column(StringUUID, nullable=False)
|
|
||||||
updated_by = db.Column(StringUUID, nullable=True)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def created_user_name(self):
|
|
||||||
account = db.session.query(Account).where(Account.id == self.created_by).first()
|
|
||||||
if account:
|
|
||||||
return account.name
|
|
||||||
return ""
|
|
||||||
|
|
||||||
|
|
||||||
class PipelineCustomizedTemplate(Base): # type: ignore[name-defined]
|
class PipelineCustomizedTemplate(Base): # type: ignore[name-defined]
|
||||||
|
|||||||
@@ -128,3 +128,22 @@ class KnowledgeConfiguration(BaseModel):
|
|||||||
if v is None:
|
if v is None:
|
||||||
return ""
|
return ""
|
||||||
return v
|
return v
|
||||||
|
|
||||||
|
|
||||||
|
class PipelineBuiltInTemplateInstallEntity(BaseModel):
|
||||||
|
icon: IconInfo | None = None
|
||||||
|
name: str
|
||||||
|
description: str
|
||||||
|
language: str
|
||||||
|
yaml_content: str
|
||||||
|
position: int = 1
|
||||||
|
|
||||||
|
|
||||||
|
class PipelineBuiltInTemplateUpdateEntity(BaseModel):
|
||||||
|
template_id: str
|
||||||
|
icon: IconInfo
|
||||||
|
name: str
|
||||||
|
description: str
|
||||||
|
language: str
|
||||||
|
yaml_content: str | None = None
|
||||||
|
position: int = 1
|
||||||
@@ -74,5 +74,4 @@ class DatabasePipelineTemplateRetrieval(PipelineTemplateRetrievalBase):
|
|||||||
"chunk_structure": pipeline_template.chunk_structure,
|
"chunk_structure": pipeline_template.chunk_structure,
|
||||||
"export_data": pipeline_template.yaml_content,
|
"export_data": pipeline_template.yaml_content,
|
||||||
"graph": graph_data,
|
"graph": graph_data,
|
||||||
"created_by": pipeline_template.created_user_name,
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,12 +8,14 @@ from datetime import UTC, datetime
|
|||||||
from typing import Any, Union, cast
|
from typing import Any, Union, cast
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import yaml
|
||||||
from flask_login import current_user
|
from flask_login import current_user
|
||||||
from sqlalchemy import func, or_, select
|
from sqlalchemy import func, or_, select, update
|
||||||
from sqlalchemy.orm import Session, sessionmaker
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
import contexts
|
import contexts
|
||||||
from configs import dify_config
|
from configs import dify_config
|
||||||
|
from controllers.console.datasets.rag_pipeline.rag_pipeline import PipelineBuiltInTemplateInstallEntity, PipelineBuiltInTemplateUpdateEntity
|
||||||
from core.app.apps.pipeline.pipeline_generator import PipelineGenerator
|
from core.app.apps.pipeline.pipeline_generator import PipelineGenerator
|
||||||
from core.app.entities.app_invoke_entities import InvokeFrom
|
from core.app.entities.app_invoke_entities import InvokeFrom
|
||||||
from core.datasource.entities.datasource_entities import (
|
from core.datasource.entities.datasource_entities import (
|
||||||
@@ -60,6 +62,7 @@ from models.dataset import ( # type: ignore
|
|||||||
Document,
|
Document,
|
||||||
DocumentPipelineExecutionLog,
|
DocumentPipelineExecutionLog,
|
||||||
Pipeline,
|
Pipeline,
|
||||||
|
PipelineBuiltInTemplate,
|
||||||
PipelineCustomizedTemplate,
|
PipelineCustomizedTemplate,
|
||||||
PipelineRecommendedPlugin,
|
PipelineRecommendedPlugin,
|
||||||
)
|
)
|
||||||
@@ -76,6 +79,7 @@ from repositories.factory import DifyAPIRepositoryFactory
|
|||||||
from services.datasource_provider_service import DatasourceProviderService
|
from services.datasource_provider_service import DatasourceProviderService
|
||||||
from services.entities.knowledge_entities.rag_pipeline_entities import (
|
from services.entities.knowledge_entities.rag_pipeline_entities import (
|
||||||
KnowledgeConfiguration,
|
KnowledgeConfiguration,
|
||||||
|
PipelineBuiltInTemplateEntity,
|
||||||
PipelineTemplateInfoEntity,
|
PipelineTemplateInfoEntity,
|
||||||
)
|
)
|
||||||
from services.errors.app import WorkflowHashNotEqualError
|
from services.errors.app import WorkflowHashNotEqualError
|
||||||
@@ -1454,3 +1458,287 @@ class RagPipelineService:
|
|||||||
if not pipeline:
|
if not pipeline:
|
||||||
raise ValueError("Pipeline not found")
|
raise ValueError("Pipeline not found")
|
||||||
return pipeline
|
return pipeline
|
||||||
|
|
||||||
|
def insert_built_in_pipeline_template(
|
||||||
|
self, args: PipelineBuiltInTemplateInstallEntity, auth_token: str
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Install built-in pipeline template
|
||||||
|
|
||||||
|
Args:
|
||||||
|
args: Pipeline built-in template entity with template metadata
|
||||||
|
file_content: YAML content of the pipeline template
|
||||||
|
auth_token: Authentication token for authorization
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: If validation fails or template processing errors occur
|
||||||
|
"""
|
||||||
|
# Validate authentication
|
||||||
|
self._validate_auth_token(auth_token)
|
||||||
|
|
||||||
|
# Parse and validate template content
|
||||||
|
pipeline_template_dsl = self._parse_template_content(args.yaml_content)
|
||||||
|
|
||||||
|
# Extract template metadata
|
||||||
|
if not args.icon:
|
||||||
|
icon = self._extract_icon_metadata(pipeline_template_dsl)
|
||||||
|
else:
|
||||||
|
icon = args.icon.model_dump()
|
||||||
|
chunk_structure = self._extract_chunk_structure(pipeline_template_dsl)
|
||||||
|
|
||||||
|
# Prepare template data
|
||||||
|
template_data = {
|
||||||
|
"name": args.name,
|
||||||
|
"description": args.description,
|
||||||
|
"chunk_structure": chunk_structure,
|
||||||
|
"icon": icon,
|
||||||
|
"language": args.language,
|
||||||
|
"yaml_content": args.yaml_content,
|
||||||
|
"position": args.position,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Use transaction for database operations
|
||||||
|
try:
|
||||||
|
self._create_new_template(template_data)
|
||||||
|
db.session.commit()
|
||||||
|
except Exception as e:
|
||||||
|
db.session.rollback()
|
||||||
|
raise ValueError(f"Failed to install pipeline template: {str(e)}")
|
||||||
|
|
||||||
|
def update_built_in_pipeline_template(self, args: PipelineBuiltInTemplateUpdateEntity, auth_token: str) -> None:
|
||||||
|
"""
|
||||||
|
Update built-in pipeline template
|
||||||
|
"""
|
||||||
|
self._validate_auth_token(auth_token)
|
||||||
|
template_data = {
|
||||||
|
"name": args.name,
|
||||||
|
"description": args.description,
|
||||||
|
"icon": args.icon.model_dump(),
|
||||||
|
"language": args.language,
|
||||||
|
"position": args.position,
|
||||||
|
}
|
||||||
|
if args.yaml_content:
|
||||||
|
template_data["yaml_content"] = args.yaml_content
|
||||||
|
chunk_structure = self._extract_chunk_structure(self._parse_template_content(args.yaml_content))
|
||||||
|
template_data["chunk_structure"] = chunk_structure
|
||||||
|
|
||||||
|
|
||||||
|
self._update_existing_template(args.template_id, template_data)
|
||||||
|
db.session.commit()
|
||||||
|
logger.info("Updated template %s with new data", args.template_id)
|
||||||
|
|
||||||
|
def _validate_auth_token(self, auth_token: str) -> None:
|
||||||
|
"""Validate the authentication token"""
|
||||||
|
config_auth_token = dify_config.UPLOAD_KNOWLEDGE_PIPELINE_TEMPLATE_TOKEN
|
||||||
|
if not config_auth_token:
|
||||||
|
raise ValueError("Auth token configuration is required")
|
||||||
|
if config_auth_token != auth_token:
|
||||||
|
raise ValueError("Auth token is incorrect")
|
||||||
|
|
||||||
|
def _parse_template_content(self, file_content: str) -> dict:
|
||||||
|
"""Parse and validate YAML template content"""
|
||||||
|
try:
|
||||||
|
pipeline_template_dsl = yaml.safe_load(file_content)
|
||||||
|
except yaml.YAMLError as e:
|
||||||
|
raise ValueError(f"Invalid YAML content: {str(e)}")
|
||||||
|
|
||||||
|
if not pipeline_template_dsl:
|
||||||
|
raise ValueError("Pipeline template DSL is required")
|
||||||
|
|
||||||
|
return pipeline_template_dsl
|
||||||
|
|
||||||
|
def _extract_icon_metadata(self, pipeline_template_dsl: dict) -> dict:
|
||||||
|
"""Extract icon metadata from template DSL"""
|
||||||
|
rag_pipeline_info = pipeline_template_dsl.get("rag_pipeline", {})
|
||||||
|
|
||||||
|
return {
|
||||||
|
"icon": rag_pipeline_info.get("icon", "📙"),
|
||||||
|
"icon_type": rag_pipeline_info.get("icon_type", "emoji"),
|
||||||
|
"icon_background": rag_pipeline_info.get("icon_background", "#FFEAD5"),
|
||||||
|
"icon_url": rag_pipeline_info.get("icon_url"),
|
||||||
|
}
|
||||||
|
|
||||||
|
def _extract_chunk_structure(self, pipeline_template_dsl: dict) -> str:
|
||||||
|
"""Extract chunk structure from template DSL"""
|
||||||
|
nodes = pipeline_template_dsl.get("workflow", {}).get("graph", {}).get("nodes", [])
|
||||||
|
|
||||||
|
# Use generator expression for efficiency
|
||||||
|
chunk_structure = next(
|
||||||
|
(
|
||||||
|
node.get("data", {}).get("chunk_structure")
|
||||||
|
for node in nodes
|
||||||
|
if node.get("data", {}).get("type") == NodeType.KNOWLEDGE_INDEX.value
|
||||||
|
),
|
||||||
|
None
|
||||||
|
)
|
||||||
|
|
||||||
|
if not chunk_structure:
|
||||||
|
raise ValueError("Chunk structure is required in template")
|
||||||
|
|
||||||
|
return chunk_structure
|
||||||
|
|
||||||
|
def _update_existing_template(self, template_id: str, template_data: dict) -> None:
|
||||||
|
"""
|
||||||
|
Update an existing pipeline template with optimistic locking
|
||||||
|
|
||||||
|
Args:
|
||||||
|
template_id: ID of the template to update
|
||||||
|
template_data: Dictionary containing updated template fields
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: If template not found
|
||||||
|
"""
|
||||||
|
# Use with_for_update() for row-level locking to prevent concurrent updates
|
||||||
|
pipeline_built_in_template = (
|
||||||
|
db.session.query(PipelineBuiltInTemplate)
|
||||||
|
.filter(PipelineBuiltInTemplate.id == template_id)
|
||||||
|
.with_for_update()
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
|
||||||
|
if not pipeline_built_in_template:
|
||||||
|
raise ValueError(f"Pipeline built-in template not found: {template_id}")
|
||||||
|
|
||||||
|
update_position = False
|
||||||
|
if template_data.get("position") != pipeline_built_in_template.position:
|
||||||
|
update_position = True
|
||||||
|
|
||||||
|
if update_position:
|
||||||
|
db.session.execute(
|
||||||
|
update(PipelineBuiltInTemplate)
|
||||||
|
.where(PipelineBuiltInTemplate.language == pipeline_built_in_template.language, PipelineBuiltInTemplate.position > pipeline_built_in_template.position)
|
||||||
|
.values(position=PipelineBuiltInTemplate.position - 1)
|
||||||
|
)
|
||||||
|
db.session.flush()
|
||||||
|
|
||||||
|
db.session.execute(
|
||||||
|
update(PipelineBuiltInTemplate)
|
||||||
|
.where(PipelineBuiltInTemplate.language == pipeline_built_in_template.language, PipelineBuiltInTemplate.position >= template_data.get("position"))
|
||||||
|
.values(position=PipelineBuiltInTemplate.position + 1)
|
||||||
|
)
|
||||||
|
db.session.flush()
|
||||||
|
|
||||||
|
# Update template fields
|
||||||
|
for key, value in template_data.items():
|
||||||
|
setattr(pipeline_built_in_template, key, value)
|
||||||
|
|
||||||
|
# Update timestamp if exists
|
||||||
|
if hasattr(pipeline_built_in_template, 'updated_at'):
|
||||||
|
pipeline_built_in_template.updated_at = datetime.now(UTC)
|
||||||
|
|
||||||
|
db.session.add(pipeline_built_in_template)
|
||||||
|
db.session.flush()
|
||||||
|
|
||||||
|
def _create_new_template(self, template_data: dict) -> None:
|
||||||
|
"""
|
||||||
|
Create a new pipeline template with atomic position assignment
|
||||||
|
|
||||||
|
Args:
|
||||||
|
template_data: Dictionary containing template fields
|
||||||
|
"""
|
||||||
|
# Use a single query with locking to get and increment position atomically
|
||||||
|
with db.session.begin_nested():
|
||||||
|
# Lock all templates of the same language to prevent position conflicts
|
||||||
|
db.session.query(PipelineBuiltInTemplate).filter(
|
||||||
|
PipelineBuiltInTemplate.language == template_data["language"]
|
||||||
|
).with_for_update().all()
|
||||||
|
|
||||||
|
# Get the next available position
|
||||||
|
position = self._get_next_position(template_data["language"])
|
||||||
|
|
||||||
|
# Add additional fields for new template
|
||||||
|
template_data.update({
|
||||||
|
"position": position,
|
||||||
|
"install_count": 0,
|
||||||
|
"copyright": dify_config.KNOWLEDGE_PIPELINE_TEMPLATE_COPYRIGHT or "",
|
||||||
|
"privacy_policy": dify_config.KNOWLEDGE_PIPELINE_TEMPLATE_PRIVACY_POLICY or "",
|
||||||
|
})
|
||||||
|
|
||||||
|
# Add timestamp if model supports it
|
||||||
|
if hasattr(PipelineBuiltInTemplate, 'created_at'):
|
||||||
|
template_data['created_at'] = datetime.now(UTC)
|
||||||
|
|
||||||
|
new_template = PipelineBuiltInTemplate(**template_data)
|
||||||
|
db.session.add(new_template)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Created new template '%s' at position %d for language %s",
|
||||||
|
template_data.get("name"), position, template_data["language"]
|
||||||
|
)
|
||||||
|
|
||||||
|
def _get_next_position(self, language: str) -> int:
|
||||||
|
"""
|
||||||
|
Get the next available position for a template in the specified language
|
||||||
|
|
||||||
|
Args:
|
||||||
|
language: Language code for the template
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Next available position number (1-based)
|
||||||
|
"""
|
||||||
|
# Use COALESCE for database compatibility
|
||||||
|
max_position = (
|
||||||
|
db.session.query(func.coalesce(func.max(PipelineBuiltInTemplate.position), 0))
|
||||||
|
.filter(PipelineBuiltInTemplate.language == language)
|
||||||
|
.scalar()
|
||||||
|
)
|
||||||
|
return max_position + 1
|
||||||
|
|
||||||
|
def uninstall_built_in_pipeline_template(self, template_id: str, auth_token: str) -> None:
|
||||||
|
"""
|
||||||
|
Uninstall a built-in pipeline template and reorder remaining templates
|
||||||
|
|
||||||
|
Args:
|
||||||
|
template_id: ID of the template to uninstall
|
||||||
|
auth_token: Authentication token for authorization
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: If template not found or authentication fails
|
||||||
|
"""
|
||||||
|
# Validate authentication
|
||||||
|
self._validate_auth_token(auth_token)
|
||||||
|
|
||||||
|
# Use transaction for atomic operations
|
||||||
|
try:
|
||||||
|
# Get the template to delete with lock to prevent concurrent modifications
|
||||||
|
pipeline_built_in_template = (
|
||||||
|
db.session.query(PipelineBuiltInTemplate)
|
||||||
|
.filter(PipelineBuiltInTemplate.id == template_id)
|
||||||
|
.with_for_update()
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
|
||||||
|
if not pipeline_built_in_template:
|
||||||
|
raise ValueError(f"Pipeline built-in template not found: {template_id}")
|
||||||
|
|
||||||
|
# Store position and language for reordering
|
||||||
|
deleted_position = pipeline_built_in_template.position
|
||||||
|
template_language = pipeline_built_in_template.language
|
||||||
|
|
||||||
|
# Delete the template first
|
||||||
|
db.session.delete(pipeline_built_in_template)
|
||||||
|
db.session.flush() # Execute delete but don't commit yet
|
||||||
|
|
||||||
|
# Batch update positions for all templates after the deleted one
|
||||||
|
# Using bulk update for better performance
|
||||||
|
db.session.execute(
|
||||||
|
update(PipelineBuiltInTemplate)
|
||||||
|
.where(
|
||||||
|
PipelineBuiltInTemplate.language == template_language,
|
||||||
|
PipelineBuiltInTemplate.position > deleted_position
|
||||||
|
)
|
||||||
|
.values(position=PipelineBuiltInTemplate.position - 1)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Commit all changes together
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Successfully uninstalled template %s at position %d for language %s",
|
||||||
|
template_id, deleted_position, template_language
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
db.session.rollback()
|
||||||
|
logger.exception("Failed to uninstall template %s", template_id)
|
||||||
|
raise ValueError(f"Failed to uninstall pipeline template: {str(e)}")
|
||||||
Reference in New Issue
Block a user