Compare commits

...

9 Commits

Author SHA1 Message Date
yessenia
7ef99293fe feat: auth init 2025-08-25 19:40:18 +08:00
autofix-ci[bot]
0aa22bf83b [autofix.ci] apply automated fixes 2025-08-20 12:20:57 +00:00
Junyan Qin
bdf70cc45a fix: mypy error 2025-08-20 20:20:18 +08:00
Junyan Qin
2c3859f39a feat(be): impl oauth server 2025-08-20 20:09:34 +08:00
Junyan Qin
a559a60e10 feat: model oauth_provider_apps 2025-08-20 14:21:14 +08:00
KVOJJJin
5f0b52c017 Fix number input in tool configure form of agent node tool item (#24154) 2025-08-19 14:26:09 +08:00
Stream
c2606f9062 fix: correct behaviour of code fix (#24152)
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
2025-08-19 14:18:49 +08:00
Asuka Minato
70da81d0e5 try ast-grep (#24149) 2025-08-19 13:41:52 +08:00
9527MrLi
75199442c1 feat: Implements periodic deletion of workflow run logs that exceed t… (#23881)
Some checks are pending
autofix.ci / autofix (push) Waiting to run
Build and Push API & Web / build (api, DIFY_API_IMAGE_NAME, linux/amd64, build-api-amd64) (push) Waiting to run
Build and Push API & Web / build (api, DIFY_API_IMAGE_NAME, linux/arm64, build-api-arm64) (push) Waiting to run
Build and Push API & Web / build (web, DIFY_WEB_IMAGE_NAME, linux/amd64, build-web-amd64) (push) Waiting to run
Build and Push API & Web / build (web, DIFY_WEB_IMAGE_NAME, linux/arm64, build-web-arm64) (push) Waiting to run
Build and Push API & Web / create-manifest (api, DIFY_API_IMAGE_NAME, merge-api-images) (push) Blocked by required conditions
Build and Push API & Web / create-manifest (web, DIFY_WEB_IMAGE_NAME, merge-web-images) (push) Blocked by required conditions
Co-authored-by: shiyun.li973792 <shiyun.li@seres.cn>
Co-authored-by: 1wangshu <suewangswu@gmail.com>
Co-authored-by: Blackoutta <hyytez@gmail.com>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
2025-08-19 09:47:34 +08:00
41 changed files with 865 additions and 27 deletions

View File

@@ -23,6 +23,9 @@ jobs:
uv run ruff check --fix-only .
# Format code
uv run ruff format .
- name: ast-grep
run: |
uvx --from ast-grep-cli sg --pattern 'db.session.query($WHATEVER).filter($HERE)' --rewrite 'db.session.query($WHATEVER).where($HERE)' -l py --update-all
- uses: autofix-ci/action@635ffb0c9798bd160680f18fd73371e355b85f27

View File

@@ -478,6 +478,13 @@ API_WORKFLOW_NODE_EXECUTION_REPOSITORY=repositories.sqlalchemy_api_workflow_node
# API workflow run repository implementation
API_WORKFLOW_RUN_REPOSITORY=repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository
# Workflow log cleanup configuration
# Enable automatic cleanup of workflow run logs to manage database size
WORKFLOW_LOG_CLEANUP_ENABLED=true
# Number of days to retain workflow run logs (default: 30 days)
WORKFLOW_LOG_RETENTION_DAYS=30
# Batch size for workflow log cleanup operations (default: 100)
WORKFLOW_LOG_CLEANUP_BATCH_SIZE=100
# App configuration
APP_MAX_EXECUTION_TIME=1200

View File

@@ -968,6 +968,14 @@ class AccountConfig(BaseSettings):
)
class WorkflowLogConfig(BaseSettings):
WORKFLOW_LOG_CLEANUP_ENABLED: bool = Field(default=True, description="Enable workflow run log cleanup")
WORKFLOW_LOG_RETENTION_DAYS: int = Field(default=30, description="Retention days for workflow run logs")
WORKFLOW_LOG_CLEANUP_BATCH_SIZE: int = Field(
default=100, description="Batch size for workflow run log cleanup operations"
)
class FeatureConfig(
# place the configs in alphabet order
AppExecutionConfig,
@@ -1003,5 +1011,6 @@ class FeatureConfig(
HostedServiceConfig,
CeleryBeatConfig,
CeleryScheduleTasksConfig,
WorkflowLogConfig,
):
pass

View File

@@ -70,7 +70,7 @@ from .app import (
)
# Import auth controllers
from .auth import activate, data_source_bearer_auth, data_source_oauth, forgot_password, login, oauth
from .auth import activate, data_source_bearer_auth, data_source_oauth, forgot_password, login, oauth, oauth_server
# Import billing controllers
from .billing import billing, compliance

View File

@@ -12,7 +12,6 @@ from controllers.console.app.error import (
)
from controllers.console.wraps import account_initialization_required, setup_required
from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
from core.helper.code_executor.code_node_provider import CodeNodeProvider
from core.helper.code_executor.javascript.javascript_code_provider import JavascriptCodeProvider
from core.helper.code_executor.python3.python3_code_provider import Python3CodeProvider
from core.llm_generator.llm_generator import LLMGenerator
@@ -126,18 +125,20 @@ class InstructionGenerateApi(Resource):
parser.add_argument("model_config", type=dict, required=True, nullable=False, location="json")
parser.add_argument("ideal_output", type=str, required=False, default="", location="json")
args = parser.parse_args()
providers: list[type[CodeNodeProvider]] = [Python3CodeProvider, JavascriptCodeProvider]
code_provider: type[CodeNodeProvider] | None = next(
(p for p in providers if p.is_accept_language(args["language"])), None
code_template = (
Python3CodeProvider.get_default_code()
if args["language"] == "python"
else (JavascriptCodeProvider.get_default_code())
if args["language"] == "javascript"
else ""
)
code_template = code_provider.get_default_code() if code_provider else ""
try:
# Generate from nothing for a workflow node
if (args["current"] == code_template or args["current"] == "") and args["node_id"] != "":
from models import App, db
from services.workflow_service import WorkflowService
app = db.session.query(App).filter(App.id == args["flow_id"]).first()
app = db.session.query(App).where(App.id == args["flow_id"]).first()
if not app:
return {"error": f"app {args['flow_id']} not found"}, 400
workflow = WorkflowService().get_draft_workflow(app_model=app)

View File

@@ -0,0 +1,189 @@
from functools import wraps
from typing import cast
import flask_login
from flask import request
from flask_restful import Resource, reqparse
from werkzeug.exceptions import BadRequest, NotFound
from controllers.console.wraps import account_initialization_required, setup_required
from core.model_runtime.utils.encoders import jsonable_encoder
from libs.login import login_required
from models.account import Account
from models.model import OAuthProviderApp
from services.oauth_server import OAUTH_ACCESS_TOKEN_EXPIRES_IN, OAuthGrantType, OAuthServerService
from .. import api
def oauth_server_client_id_required(view):
@wraps(view)
def decorated(*args, **kwargs):
parser = reqparse.RequestParser()
parser.add_argument("client_id", type=str, required=True, location="json")
parsed_args = parser.parse_args()
client_id = parsed_args.get("client_id")
if not client_id:
raise BadRequest("client_id is required")
oauth_provider_app = OAuthServerService.get_oauth_provider_app(client_id)
if not oauth_provider_app:
raise NotFound("client_id is invalid")
kwargs["oauth_provider_app"] = oauth_provider_app
return view(*args, **kwargs)
return decorated
def oauth_server_access_token_required(view):
@wraps(view)
def decorated(*args, **kwargs):
oauth_provider_app = kwargs.get("oauth_provider_app")
if not oauth_provider_app or not isinstance(oauth_provider_app, OAuthProviderApp):
raise BadRequest("Invalid oauth_provider_app")
if not request.headers.get("Authorization"):
raise BadRequest("Authorization is required")
authorization_header = request.headers.get("Authorization")
if not authorization_header:
raise BadRequest("Authorization header is required")
parts = authorization_header.split(" ")
if len(parts) != 2:
raise BadRequest("Invalid Authorization header format")
token_type = parts[0]
if token_type != "Bearer":
raise BadRequest("token_type is invalid")
access_token = parts[1]
if not access_token:
raise BadRequest("access_token is required")
account = OAuthServerService.validate_oauth_access_token(oauth_provider_app.client_id, access_token)
if not account:
raise BadRequest("access_token or client_id is invalid")
kwargs["account"] = account
return view(*args, **kwargs)
return decorated
class OAuthServerAppApi(Resource):
@setup_required
@oauth_server_client_id_required
def post(self, oauth_provider_app: OAuthProviderApp):
parser = reqparse.RequestParser()
parser.add_argument("redirect_uri", type=str, required=True, location="json")
parsed_args = parser.parse_args()
redirect_uri = parsed_args.get("redirect_uri")
# check if redirect_uri is valid
if redirect_uri not in oauth_provider_app.redirect_uris:
raise BadRequest("redirect_uri is invalid")
return jsonable_encoder(
{
"app_icon": oauth_provider_app.app_icon,
"app_label": oauth_provider_app.app_label,
"scope": oauth_provider_app.scope,
}
)
class OAuthServerUserAuthorizeApi(Resource):
@setup_required
@login_required
@account_initialization_required
@oauth_server_client_id_required
def post(self, oauth_provider_app: OAuthProviderApp):
account = cast(Account, flask_login.current_user)
user_account_id = account.id
code = OAuthServerService.sign_oauth_authorization_code(oauth_provider_app.client_id, user_account_id)
return jsonable_encoder(
{
"code": code,
}
)
class OAuthServerUserTokenApi(Resource):
@setup_required
@oauth_server_client_id_required
def post(self, oauth_provider_app: OAuthProviderApp):
parser = reqparse.RequestParser()
parser.add_argument("grant_type", type=str, required=True, location="json")
parser.add_argument("code", type=str, required=False, location="json")
parser.add_argument("client_secret", type=str, required=False, location="json")
parser.add_argument("redirect_uri", type=str, required=False, location="json")
parser.add_argument("refresh_token", type=str, required=False, location="json")
parsed_args = parser.parse_args()
grant_type = OAuthGrantType(parsed_args["grant_type"])
if grant_type == OAuthGrantType.AUTHORIZATION_CODE:
if not parsed_args["code"]:
raise BadRequest("code is required")
if parsed_args["client_secret"] != oauth_provider_app.client_secret:
raise BadRequest("client_secret is invalid")
if parsed_args["redirect_uri"] not in oauth_provider_app.redirect_uris:
raise BadRequest("redirect_uri is invalid")
access_token, refresh_token = OAuthServerService.sign_oauth_access_token(
grant_type, code=parsed_args["code"], client_id=oauth_provider_app.client_id
)
return jsonable_encoder(
{
"access_token": access_token,
"token_type": "Bearer",
"expires_in": OAUTH_ACCESS_TOKEN_EXPIRES_IN,
"refresh_token": refresh_token,
}
)
elif grant_type == OAuthGrantType.REFRESH_TOKEN:
if not parsed_args["refresh_token"]:
raise BadRequest("refresh_token is required")
access_token, refresh_token = OAuthServerService.sign_oauth_access_token(
grant_type, refresh_token=parsed_args["refresh_token"], client_id=oauth_provider_app.client_id
)
return jsonable_encoder(
{
"access_token": access_token,
"token_type": "Bearer",
"expires_in": OAUTH_ACCESS_TOKEN_EXPIRES_IN,
"refresh_token": refresh_token,
}
)
else:
raise BadRequest("invalid grant_type")
class OAuthServerUserAccountApi(Resource):
@setup_required
@oauth_server_client_id_required
@oauth_server_access_token_required
def post(self, oauth_provider_app: OAuthProviderApp, account: Account):
return jsonable_encoder(
{
"name": account.name,
"email": account.email,
"avatar": account.avatar,
"interface_language": account.interface_language,
"timezone": account.timezone,
}
)
api.add_resource(OAuthServerAppApi, "/oauth/provider")
api.add_resource(OAuthServerUserAuthorizeApi, "/oauth/provider/authorize")
api.add_resource(OAuthServerUserTokenApi, "/oauth/provider/token")
api.add_resource(OAuthServerUserAccountApi, "/oauth/provider/account")

View File

@@ -39,7 +39,7 @@ class UploadFileApi(Resource):
data_source_info = document.data_source_info_dict
if data_source_info and "upload_file_id" in data_source_info:
file_id = data_source_info["upload_file_id"]
upload_file = db.session.query(UploadFile).filter(UploadFile.id == file_id).first()
upload_file = db.session.query(UploadFile).where(UploadFile.id == file_id).first()
if not upload_file:
raise NotFound("UploadFile not found.")
else:

View File

@@ -181,7 +181,7 @@ class MessageCycleManager:
:param message_id: message id
:return:
"""
message_file = db.session.query(MessageFile).filter(MessageFile.id == message_id).first()
message_file = db.session.query(MessageFile).where(MessageFile.id == message_id).first()
event_type = StreamEvent.MESSAGE_FILE if message_file else StreamEvent.MESSAGE
return MessageStreamResponse(

View File

@@ -399,9 +399,9 @@ class LLMGenerator:
def instruction_modify_legacy(
tenant_id: str, flow_id: str, current: str, instruction: str, model_config: dict, ideal_output: str | None
) -> dict:
app: App | None = db.session.query(App).filter(App.id == flow_id).first()
app: App | None = db.session.query(App).where(App.id == flow_id).first()
last_run: Message | None = (
db.session.query(Message).filter(Message.app_id == flow_id).order_by(Message.created_at.desc()).first()
db.session.query(Message).where(Message.app_id == flow_id).order_by(Message.created_at.desc()).first()
)
if not last_run:
return LLMGenerator.__instruction_modify_common(
@@ -442,7 +442,7 @@ class LLMGenerator:
) -> dict:
from services.workflow_service import WorkflowService
app: App | None = db.session.query(App).filter(App.id == flow_id).first()
app: App | None = db.session.query(App).where(App.id == flow_id).first()
if not app:
raise ValueError("App not found.")
workflow = WorkflowService().get_draft_workflow(app_model=app)

View File

@@ -414,7 +414,7 @@ When you are modifying the code, you should remember:
- Get inputs from the parameters of the function and have explicit type annotations.
- Write proper imports at the top of the code.
- Use return statement to return the result.
- You should return a `dict`.
- You should return a `dict`. If you need to return a `result: str`, you should `return {"result": result}`.
Your output must strictly follow the schema format, do not output any content outside of the JSON body.
""" # noqa: E501

View File

@@ -151,7 +151,13 @@ def init_app(app: DifyApp) -> Celery:
"task": "schedule.check_upgradable_plugin_task.check_upgradable_plugin_task",
"schedule": crontab(minute="*/15"),
}
if dify_config.WORKFLOW_LOG_CLEANUP_ENABLED:
# 2:00 AM every day
imports.append("schedule.clean_workflow_runlogs_precise")
beat_schedule["clean_workflow_runlogs_precise"] = {
"task": "schedule.clean_workflow_runlogs_precise.clean_workflow_runlogs_precise",
"schedule": crontab(minute="0", hour="2"),
}
celery_app.conf.update(beat_schedule=beat_schedule, imports=imports)
return celery_app

View File

@@ -0,0 +1,45 @@
"""empty message
Revision ID: 8d289573e1da
Revises: fa8b0fa6f407
Create Date: 2025-08-20 17:47:17.015695
"""
from alembic import op
import models as models
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '8d289573e1da'
down_revision = 'fa8b0fa6f407'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('oauth_provider_apps',
sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
sa.Column('app_icon', sa.String(length=255), nullable=False),
sa.Column('app_label', sa.JSON(), server_default='{}', nullable=False),
sa.Column('client_id', sa.String(length=255), nullable=False),
sa.Column('client_secret', sa.String(length=255), nullable=False),
sa.Column('redirect_uris', sa.JSON(), server_default='[]', nullable=False),
sa.Column('scope', sa.String(length=255), server_default=sa.text("'read:name read:email read:avatar read:interface_language read:timezone'"), nullable=False),
sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False),
sa.PrimaryKeyConstraint('id', name='oauth_provider_app_pkey')
)
with op.batch_alter_table('oauth_provider_apps', schema=None) as batch_op:
batch_op.create_index('oauth_provider_app_client_id_idx', ['client_id'], unique=False)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('oauth_provider_apps', schema=None) as batch_op:
batch_op.drop_index('oauth_provider_app_client_id_idx')
op.drop_table('oauth_provider_apps')
# ### end Alembic commands ###

View File

@@ -607,6 +607,32 @@ class InstalledApp(Base):
return tenant
class OAuthProviderApp(Base):
"""
Globally shared OAuth provider app information.
Only for Dify Cloud.
"""
__tablename__ = "oauth_provider_apps"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="oauth_provider_app_pkey"),
sa.Index("oauth_provider_app_client_id_idx", "client_id"),
)
id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
app_icon = mapped_column(String(255), nullable=False)
app_label = mapped_column(sa.JSON, nullable=False, server_default="{}")
client_id = mapped_column(String(255), nullable=False)
client_secret = mapped_column(String(255), nullable=False)
redirect_uris = mapped_column(sa.JSON, nullable=False, server_default="[]")
scope = mapped_column(
String(255),
nullable=False,
server_default=sa.text("'read:name read:email read:avatar read:interface_language read:timezone'"),
)
created_at = mapped_column(sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)"))
class Conversation(Base):
__tablename__ = "conversations"
__table_args__ = (

View File

@@ -0,0 +1,155 @@
import datetime
import logging
import time
import click
import app
from configs import dify_config
from extensions.ext_database import db
from models.model import (
AppAnnotationHitHistory,
Conversation,
Message,
MessageAgentThought,
MessageAnnotation,
MessageChain,
MessageFeedback,
MessageFile,
)
from models.workflow import ConversationVariable, WorkflowAppLog, WorkflowNodeExecutionModel, WorkflowRun
_logger = logging.getLogger(__name__)
MAX_RETRIES = 3
BATCH_SIZE = dify_config.WORKFLOW_LOG_CLEANUP_BATCH_SIZE
@app.celery.task(queue="dataset")
def clean_workflow_runlogs_precise():
"""Clean expired workflow run logs with retry mechanism and complete message cascade"""
click.echo(click.style("Start clean workflow run logs (precise mode with complete cascade).", fg="green"))
start_at = time.perf_counter()
retention_days = dify_config.WORKFLOW_LOG_RETENTION_DAYS
cutoff_date = datetime.datetime.now() - datetime.timedelta(days=retention_days)
try:
total_workflow_runs = db.session.query(WorkflowRun).where(WorkflowRun.created_at < cutoff_date).count()
if total_workflow_runs == 0:
_logger.info("No expired workflow run logs found")
return
_logger.info("Found %s expired workflow run logs to clean", total_workflow_runs)
total_deleted = 0
failed_batches = 0
batch_count = 0
while True:
workflow_runs = (
db.session.query(WorkflowRun.id).where(WorkflowRun.created_at < cutoff_date).limit(BATCH_SIZE).all()
)
if not workflow_runs:
break
workflow_run_ids = [run.id for run in workflow_runs]
batch_count += 1
success = _delete_batch_with_retry(workflow_run_ids, failed_batches)
if success:
total_deleted += len(workflow_run_ids)
failed_batches = 0
else:
failed_batches += 1
if failed_batches >= MAX_RETRIES:
_logger.error("Failed to delete batch after %s retries, aborting cleanup for today", MAX_RETRIES)
break
else:
# Calculate incremental delay times: 5, 10, 15 minutes
retry_delay_minutes = failed_batches * 5
_logger.warning("Batch deletion failed, retrying in %s minutes...", retry_delay_minutes)
time.sleep(retry_delay_minutes * 60)
continue
_logger.info("Cleanup completed: %s expired workflow run logs deleted", total_deleted)
except Exception as e:
db.session.rollback()
_logger.exception("Unexpected error in workflow log cleanup")
raise
end_at = time.perf_counter()
execution_time = end_at - start_at
click.echo(click.style(f"Cleaned workflow run logs from db success latency: {execution_time:.2f}s", fg="green"))
def _delete_batch_with_retry(workflow_run_ids: list[str], attempt_count: int) -> bool:
"""Delete a single batch with a retry mechanism and complete cascading deletion"""
try:
with db.session.begin_nested():
message_data = (
db.session.query(Message.id, Message.conversation_id)
.filter(Message.workflow_run_id.in_(workflow_run_ids))
.all()
)
message_id_list = [msg.id for msg in message_data]
conversation_id_list = list({msg.conversation_id for msg in message_data if msg.conversation_id})
if message_id_list:
db.session.query(AppAnnotationHitHistory).where(
AppAnnotationHitHistory.message_id.in_(message_id_list)
).delete(synchronize_session=False)
db.session.query(MessageAgentThought).where(MessageAgentThought.message_id.in_(message_id_list)).delete(
synchronize_session=False
)
db.session.query(MessageChain).where(MessageChain.message_id.in_(message_id_list)).delete(
synchronize_session=False
)
db.session.query(MessageFile).where(MessageFile.message_id.in_(message_id_list)).delete(
synchronize_session=False
)
db.session.query(MessageAnnotation).where(MessageAnnotation.message_id.in_(message_id_list)).delete(
synchronize_session=False
)
db.session.query(MessageFeedback).where(MessageFeedback.message_id.in_(message_id_list)).delete(
synchronize_session=False
)
db.session.query(Message).where(Message.workflow_run_id.in_(workflow_run_ids)).delete(
synchronize_session=False
)
db.session.query(WorkflowAppLog).where(WorkflowAppLog.workflow_run_id.in_(workflow_run_ids)).delete(
synchronize_session=False
)
db.session.query(WorkflowNodeExecutionModel).where(
WorkflowNodeExecutionModel.workflow_run_id.in_(workflow_run_ids)
).delete(synchronize_session=False)
if conversation_id_list:
db.session.query(ConversationVariable).where(
ConversationVariable.conversation_id.in_(conversation_id_list)
).delete(synchronize_session=False)
db.session.query(Conversation).where(Conversation.id.in_(conversation_id_list)).delete(
synchronize_session=False
)
db.session.query(WorkflowRun).where(WorkflowRun.id.in_(workflow_run_ids)).delete(synchronize_session=False)
db.session.commit()
return True
except Exception as e:
db.session.rollback()
_logger.exception("Batch deletion failed (attempt %s)", attempt_count + 1)
return False

View File

@@ -293,7 +293,7 @@ class AppAnnotationService:
annotation_ids_to_delete = [annotation.id for annotation, _ in annotations_to_delete]
# Step 2: Bulk delete hit histories in a single query
db.session.query(AppAnnotationHitHistory).filter(
db.session.query(AppAnnotationHitHistory).where(
AppAnnotationHitHistory.annotation_id.in_(annotation_ids_to_delete)
).delete(synchronize_session=False)
@@ -307,7 +307,7 @@ class AppAnnotationService:
# Step 4: Bulk delete annotations in a single query
deleted_count = (
db.session.query(MessageAnnotation)
.filter(MessageAnnotation.id.in_(annotation_ids_to_delete))
.where(MessageAnnotation.id.in_(annotation_ids_to_delete))
.delete(synchronize_session=False)
)
@@ -505,9 +505,9 @@ class AppAnnotationService:
db.session.query(AppAnnotationSetting).where(AppAnnotationSetting.app_id == app_id).first()
)
annotations_query = db.session.query(MessageAnnotation).filter(MessageAnnotation.app_id == app_id)
annotations_query = db.session.query(MessageAnnotation).where(MessageAnnotation.app_id == app_id)
for annotation in annotations_query.yield_per(100):
annotation_hit_histories_query = db.session.query(AppAnnotationHitHistory).filter(
annotation_hit_histories_query = db.session.query(AppAnnotationHitHistory).where(
AppAnnotationHitHistory.annotation_id == annotation.id
)
for annotation_hit_history in annotation_hit_histories_query.yield_per(100):

View File

@@ -0,0 +1,94 @@
import enum
import uuid
from sqlalchemy import select
from sqlalchemy.orm import Session
from werkzeug.exceptions import BadRequest
from extensions.ext_database import db
from extensions.ext_redis import redis_client
from models.account import Account
from models.model import OAuthProviderApp
from services.account_service import AccountService
class OAuthGrantType(enum.StrEnum):
AUTHORIZATION_CODE = "authorization_code"
REFRESH_TOKEN = "refresh_token"
OAUTH_AUTHORIZATION_CODE_REDIS_KEY = "oauth_provider:{client_id}:authorization_code:{code}"
OAUTH_ACCESS_TOKEN_REDIS_KEY = "oauth_provider:{client_id}:access_token:{token}"
OAUTH_ACCESS_TOKEN_EXPIRES_IN = 60 * 60 * 12 # 12 hours
OAUTH_REFRESH_TOKEN_REDIS_KEY = "oauth_provider:{client_id}:refresh_token:{token}"
OAUTH_REFRESH_TOKEN_EXPIRES_IN = 60 * 60 * 24 * 30 # 30 days
class OAuthServerService:
@staticmethod
def get_oauth_provider_app(client_id: str) -> OAuthProviderApp | None:
query = select(OAuthProviderApp).where(OAuthProviderApp.client_id == client_id)
with Session(db.engine) as session:
return session.execute(query).scalar_one_or_none()
@staticmethod
def sign_oauth_authorization_code(client_id: str, user_account_id: str) -> str:
code = str(uuid.uuid4())
redis_key = OAUTH_AUTHORIZATION_CODE_REDIS_KEY.format(client_id=client_id, code=code)
redis_client.set(redis_key, user_account_id, ex=60 * 10) # 10 minutes
return code
@staticmethod
def sign_oauth_access_token(
grant_type: OAuthGrantType,
code: str = "",
client_id: str = "",
refresh_token: str = "",
) -> tuple[str, str]:
match grant_type:
case OAuthGrantType.AUTHORIZATION_CODE:
redis_key = OAUTH_AUTHORIZATION_CODE_REDIS_KEY.format(client_id=client_id, code=code)
user_account_id = redis_client.get(redis_key)
if not user_account_id:
raise BadRequest("invalid code")
# delete code
redis_client.delete(redis_key)
access_token = OAuthServerService._sign_oauth_access_token(client_id, user_account_id)
refresh_token = OAuthServerService._sign_oauth_refresh_token(client_id, user_account_id)
return access_token, refresh_token
case OAuthGrantType.REFRESH_TOKEN:
redis_key = OAUTH_REFRESH_TOKEN_REDIS_KEY.format(client_id=client_id, token=refresh_token)
user_account_id = redis_client.get(redis_key)
if not user_account_id:
raise BadRequest("invalid refresh token")
access_token = OAuthServerService._sign_oauth_access_token(client_id, user_account_id)
return access_token, refresh_token
@staticmethod
def _sign_oauth_access_token(client_id: str, user_account_id: str) -> str:
token = str(uuid.uuid4())
redis_key = OAUTH_ACCESS_TOKEN_REDIS_KEY.format(client_id=client_id, token=token)
redis_client.set(redis_key, user_account_id, ex=OAUTH_ACCESS_TOKEN_EXPIRES_IN)
return token
@staticmethod
def _sign_oauth_refresh_token(client_id: str, user_account_id: str) -> str:
token = str(uuid.uuid4())
redis_key = OAUTH_REFRESH_TOKEN_REDIS_KEY.format(client_id=client_id, token=token)
redis_client.set(redis_key, user_account_id, ex=OAUTH_REFRESH_TOKEN_EXPIRES_IN)
return token
@staticmethod
def validate_oauth_access_token(client_id: str, token: str) -> Account | None:
redis_key = OAUTH_ACCESS_TOKEN_REDIS_KEY.format(client_id=client_id, token=token)
user_account_id = redis_client.get(redis_key)
if not user_account_id:
return None
user_id_str = user_account_id.decode("utf-8")
return AccountService.load_user(user_id_str)

View File

@@ -471,7 +471,7 @@ class TestAnnotationService:
# Verify annotation was deleted
from extensions.ext_database import db
deleted_annotation = db.session.query(MessageAnnotation).filter(MessageAnnotation.id == annotation_id).first()
deleted_annotation = db.session.query(MessageAnnotation).where(MessageAnnotation.id == annotation_id).first()
assert deleted_annotation is None
# Verify delete_annotation_index_task was called (when annotation setting exists)
@@ -1175,7 +1175,7 @@ class TestAnnotationService:
AppAnnotationService.delete_app_annotation(app.id, annotation_id)
# Verify annotation was deleted
deleted_annotation = db.session.query(MessageAnnotation).filter(MessageAnnotation.id == annotation_id).first()
deleted_annotation = db.session.query(MessageAnnotation).where(MessageAnnotation.id == annotation_id).first()
assert deleted_annotation is None
# Verify delete_annotation_index_task was called

View File

@@ -234,7 +234,7 @@ class TestAPIBasedExtensionService:
# Verify extension was deleted
from extensions.ext_database import db
deleted_extension = db.session.query(APIBasedExtension).filter(APIBasedExtension.id == extension_id).first()
deleted_extension = db.session.query(APIBasedExtension).where(APIBasedExtension.id == extension_id).first()
assert deleted_extension is None
def test_save_extension_duplicate_name(self, db_session_with_containers, mock_external_service_dependencies):

View File

@@ -484,7 +484,7 @@ class TestMessageService:
# Verify feedback was deleted
from extensions.ext_database import db
deleted_feedback = db.session.query(MessageFeedback).filter(MessageFeedback.id == feedback.id).first()
deleted_feedback = db.session.query(MessageFeedback).where(MessageFeedback.id == feedback.id).first()
assert deleted_feedback is None
def test_create_feedback_no_rating_when_not_exists(

View File

@@ -469,6 +469,6 @@ class TestModelLoadBalancingService:
# Verify inherit config was created in database
inherit_configs = (
db.session.query(LoadBalancingModelConfig).filter(LoadBalancingModelConfig.name == "__inherit__").all()
db.session.query(LoadBalancingModelConfig).where(LoadBalancingModelConfig.name == "__inherit__").all()
)
assert len(inherit_configs) == 1

View File

@@ -887,6 +887,14 @@ API_WORKFLOW_RUN_REPOSITORY=repositories.sqlalchemy_api_workflow_run_repository.
# API workflow node execution repository implementation
API_WORKFLOW_NODE_EXECUTION_REPOSITORY=repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository
# Workflow log cleanup configuration
# Enable automatic cleanup of workflow run logs to manage database size
WORKFLOW_LOG_CLEANUP_ENABLED=false
# Number of days to retain workflow run logs (default: 30 days)
WORKFLOW_LOG_RETENTION_DAYS=30
# Batch size for workflow log cleanup operations (default: 100)
WORKFLOW_LOG_CLEANUP_BATCH_SIZE=100
# HTTP request node in workflow configuration
HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760
HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576

View File

@@ -396,6 +396,9 @@ x-shared-env: &shared-api-worker-env
CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY: ${CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY:-core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository}
API_WORKFLOW_RUN_REPOSITORY: ${API_WORKFLOW_RUN_REPOSITORY:-repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository}
API_WORKFLOW_NODE_EXECUTION_REPOSITORY: ${API_WORKFLOW_NODE_EXECUTION_REPOSITORY:-repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository}
WORKFLOW_LOG_CLEANUP_ENABLED: ${WORKFLOW_LOG_CLEANUP_ENABLED:-false}
WORKFLOW_LOG_RETENTION_DAYS: ${WORKFLOW_LOG_RETENTION_DAYS:-30}
WORKFLOW_LOG_CLEANUP_BATCH_SIZE: ${WORKFLOW_LOG_CLEANUP_BATCH_SIZE:-100}
HTTP_REQUEST_NODE_MAX_BINARY_SIZE: ${HTTP_REQUEST_NODE_MAX_BINARY_SIZE:-10485760}
HTTP_REQUEST_NODE_MAX_TEXT_SIZE: ${HTTP_REQUEST_NODE_MAX_TEXT_SIZE:-1048576}
HTTP_REQUEST_NODE_SSL_VERIFY: ${HTTP_REQUEST_NODE_SSL_VERIFY:-True}

View File

@@ -2,11 +2,11 @@
import { useTranslation } from 'react-i18next'
import { RiArrowRightUpLine, RiRobot2Line } from '@remixicon/react'
import { useRouter } from 'next/navigation'
import Button from '../components/base/button'
import Avatar from './avatar'
import Button from '@/app/components/base/button'
import DifyLogo from '@/app/components/base/logo/dify-logo'
import { useCallback } from 'react'
import { useGlobalPublicStore } from '@/context/global-public-context'
import Avatar from './avatar'
const Header = () => {
const { t } = useTranslation()

View File

@@ -0,0 +1,38 @@
'use client'
import Header from '@/app/signin/_header'
import cn from '@/utils/classnames'
import { useGlobalPublicStore } from '@/context/global-public-context'
import useDocumentTitle from '@/hooks/use-document-title'
import { useEffect, useState } from 'react'
import { AppContextProvider } from '@/context/app-context'
export default function SignInLayout({ children }: any) {
const { systemFeatures } = useGlobalPublicStore()
useDocumentTitle('')
const [isLoggedIn, setIsLoggedIn] = useState(false)
useEffect(() => {
setIsLoggedIn(!!localStorage.getItem('console_token'))
}, [])
return <>
<div className={cn('flex min-h-screen w-full justify-center bg-background-default-burn p-6')}>
<div className={cn('flex w-full shrink-0 flex-col rounded-2xl border border-effects-highlight bg-background-default-subtle')}>
<Header />
<div className={cn('flex w-full grow flex-col items-center justify-center px-6 md:px-[108px]')}>
<div className='flex flex-col md:w-[400px]'>
{
isLoggedIn
? <AppContextProvider>
{children}
</AppContextProvider>
: children
}
</div>
</div>
{systemFeatures.branding.enabled === false && <div className='system-xs-regular px-8 py-6 text-text-tertiary'>
© {new Date().getFullYear()} LangGenius, Inc. All rights reserved.
</div>}
</div>
</div>
</>
}

View File

@@ -0,0 +1,185 @@
'use client'
import React, { useMemo } from 'react'
import { useRouter, useSearchParams } from 'next/navigation'
import cn from '@/utils/classnames'
import Button from '@/app/components/base/button'
import Avatar from '@/app/components/base/avatar'
import { useAppContext } from '@/context/app-context'
import { useAuthorizeOAuthApp, useOAuthAppInfo } from '@/service/use-oauth-provider'
import Loading from '@/app/components/base/loading'
import {
RiAccountCircleLine,
RiGlobalLine,
RiInfoCardLine,
RiMailLine,
RiTranslate2,
} from '@remixicon/react'
const SCOPE_ICON_MAP: Record<string, { icon: React.ComponentType<{ className?: string }>, label: string }> = {
'read:name': {
icon: RiInfoCardLine,
label: 'Name',
},
'read:email': {
icon: RiMailLine,
label: 'Email',
},
'read:avatar': {
icon: RiAccountCircleLine,
label: 'Avatar',
},
'read:interface_language': {
icon: RiTranslate2,
label: 'Language Preference',
},
'read:timezone': {
icon: RiGlobalLine,
label: 'Timezone',
},
}
const STORAGE_KEY = 'oauth_authorize_pending'
function buildReturnUrl(pathname: string, search: string) {
try {
const base = `${globalThis.location.origin}${pathname}${search}`
return base
}
catch {
return pathname + search
}
}
export default function OAuthAuthorizePage() {
const router = useRouter()
const searchParams = useSearchParams()
const client_id = searchParams.get('client_id') || ''
const redirect_uri = searchParams.get('redirect_uri') || ''
const response_type = searchParams.get('response_type') || 'code'
const { userProfile } = useAppContext()
const { data: authAppInfo, isLoading, isError, error } = useOAuthAppInfo(client_id, redirect_uri, true)
const { mutateAsync: authorize, isPending: authorizing } = useAuthorizeOAuthApp()
const isLoggedIn = useMemo(() => {
try {
return Boolean(localStorage.getItem('console_token'))
}
catch { return false }
}, [])
const invalidParams = !client_id || !redirect_uri || response_type !== 'code'
const onLoginClick = () => {
try {
const returnUrl = buildReturnUrl('/account/oauth/authorize', `?client_id=${encodeURIComponent(client_id)}&redirect_uri=${encodeURIComponent(redirect_uri)}`)
localStorage.setItem(STORAGE_KEY, JSON.stringify({ client_id, redirect_uri, returnUrl }))
router.push(`/signin?redirect_url=${encodeURIComponent(returnUrl)}`)
}
catch {
router.push('/signin')
}
}
const onAuthorize = async () => {
if (!client_id || !redirect_uri)
return
try {
const { code } = await authorize({ client_id })
const url = new URL(redirect_uri)
url.searchParams.set('code', code)
globalThis.location.href = url.toString()
}
catch {
// handled by global toast
}
}
if (isLoading) {
return (
<div className='bg-background-default-subtle'>
<Loading type='app' />
</div>
)
}
if (invalidParams || isError) {
return (
<div className={cn('mx-auto mt-8 w-full px-6 md:px-[108px]')}>
<p className='body-md-regular mt-2 text-text-tertiary'>{(error as any)?.message || 'Invalid parameters'}</p>
</div>
)
}
return (
<div className='bg-background-default-subtle'>
{authAppInfo?.app_icon && (
<div className='w-max rounded-2xl border-[0.5px] border-components-panel-border bg-text-primary-on-surface p-3 shadow-lg'>
{/* <img src={authAppInfo.app_icon} alt='app icon' className='h-10 w-10 rounded' /> */}
<img src={'https://cloud.dify.ai/console/api/workspaces/current/tool-provider/builtin/time/icon'} alt='app icon' className='h-10 w-10 rounded' />
</div>
)}
<div className={`mb-4 mt-5 flex flex-col gap-2 ${isLoggedIn ? 'pb-2' : ''}`}>
<div className='title-4xl-semi-bold'>
{isLoggedIn && <div className='text-text-primary'>Connect to</div>}
<div className='text-[var(--color-saas-dify-blue-inverted)]'>{authAppInfo?.app_label?.en_US || authAppInfo?.app_label?.zh_Hans || authAppInfo?.app_label?.ja_JP}</div>
{!isLoggedIn && <div className='text-text-primary'>wants to access your Dify Cloud account</div>}
</div>
<div className='body-md-regular text-text-secondary'>{isLoggedIn ? `${authAppInfo?.app_label?.en_US} wants to access your Dify account` : 'Please log in to authorize'}</div>
</div>
{isLoggedIn && (
<div className='flex items-center justify-between rounded-xl bg-background-section-burn p-3'>
<div className='flex items-center gap-2.5'>
<Avatar avatar={userProfile.avatar_url} name={userProfile.name} size={36} />
<div>
<div className='system-md-semi-bold text-text-secondary'>{userProfile.name}</div>
<div className='system-xs-regular text-text-tertiary'>{userProfile.email}</div>
</div>
</div>
<Button variant='tertiary' size='small' onClick={() => router.push('/signin')}>Switch account</Button>
</div>
)}
{isLoggedIn && Boolean(authAppInfo?.scope) && (
<div className='mt-2 flex flex-col gap-2.5 rounded-xl bg-background-section-burn px-[22px] py-5 text-text-secondary'>
{authAppInfo!.scope.split(/\s+/).filter(Boolean).map((scope: string) => {
const Icon = SCOPE_ICON_MAP[scope]
return (
<div key={scope} className='body-sm-medium flex items-center gap-2 text-text-secondary'>
{Icon ? <Icon.icon className='h-4 w-4' /> : <RiAccountCircleLine className='h-4 w-4' />}
{Icon.label}
</div>
)
})}
</div>
)}
<div className='flex flex-col items-center gap-2 pt-4'>
{!isLoggedIn ? (
<Button variant='primary' size='large' className='w-full' onClick={onLoginClick}>Login</Button>
) : (
<>
<Button variant='primary' size='large' className='w-full' onClick={onAuthorize} disabled={authorizing} loading={authorizing}>Authorize</Button>
<Button size='large' className='w-full' onClick={() => router.push('/apps')}>Cancel</Button>
</>
)}
</div>
<div className='mt-4 py-2'>
<svg xmlns="http://www.w3.org/2000/svg" width="400" height="1" viewBox="0 0 400 1" fill="none">
<path d="M0 0.5H400" stroke="url(#paint0_linear_2_5904)" />
<defs>
<linearGradient id="paint0_linear_2_5904" x1="400" y1="9.49584" x2="0.000228929" y2="9.17666" gradientUnits="userSpaceOnUse">
<stop stop-color="white" stop-opacity="0.01" />
<stop offset="0.505" stop-color="#101828" stop-opacity="0.08" />
<stop offset="1" stop-color="white" stop-opacity="0.01" />
</linearGradient>
</defs>
</svg>
</div>
<div className='system-xs-regular mt-3 text-text-tertiary'>We respect your privacy and will only use this information to enhance your experience with our developer tools.</div>
</div>
)
}

View File

@@ -142,7 +142,7 @@ export const GetCodeGeneratorResModal: FC<IGetCodeGeneratorResProps> = (
ideal_output: ideaOutput,
language: languageMap[codeLanguages] || 'javascript',
})
if(!currentCode)
if((res as any).code) // not current or current is the same as the template would return a code field
res.modified = (res as any).code
if (error) {

View File

@@ -259,7 +259,7 @@ const ReasoningConfigForm: React.FC<Props> = ({
className='h-8 grow'
type='number'
value={varInput?.value || ''}
onChange={handleValueChange(variable, type)}
onChange={e => handleValueChange(variable, type)(e.target.value)}
placeholder={placeholder?.[language] || placeholder?.en_US}
/>
)}

View File

@@ -21,6 +21,7 @@ const NormalForm = () => {
const searchParams = useSearchParams()
const consoleToken = decodeURIComponent(searchParams.get('access_token') || '')
const refreshToken = decodeURIComponent(searchParams.get('refresh_token') || '')
const redirectUrl = searchParams.get('redirect_url') || ''
const message = decodeURIComponent(searchParams.get('message') || '')
const invite_token = decodeURIComponent(searchParams.get('invite_token') || '')
const [isLoading, setIsLoading] = useState(true)
@@ -37,6 +38,22 @@ const NormalForm = () => {
if (consoleToken && refreshToken) {
localStorage.setItem('console_token', consoleToken)
localStorage.setItem('refresh_token', refreshToken)
const pendingStr = localStorage.getItem('oauth_authorize_pending')
if (redirectUrl) {
router.replace(decodeURIComponent(redirectUrl))
return
}
if (pendingStr) {
try {
const pending = JSON.parse(pendingStr)
if (pending?.returnUrl) {
localStorage.removeItem('oauth_authorize_pending')
router.replace(pending.returnUrl)
return
}
}
catch {}
}
router.replace('/apps')
return
}

View File

@@ -0,0 +1,23 @@
import { post } from './base'
export type OAuthAppInfo = {
app_icon: string
app_label: Record<string, string>
scope: string
}
export type OAuthAuthorizeResponse = {
code: string
}
export async function fetchOAuthAppInfo(client_id: string, redirect_uri: string) {
return post<OAuthAppInfo>('/oauth/provider', {
body: { client_id, redirect_uri },
}, { silent: true })
}
export async function authorizeOAuthApp(client_id: string) {
return post<OAuthAuthorizeResponse>('/oauth/provider/authorize', {
body: { client_id },
})
}

View File

@@ -0,0 +1,29 @@
import { post } from './base'
import { useMutation, useQuery } from '@tanstack/react-query'
const NAME_SPACE = 'oauth-provider'
export type OAuthAppInfo = {
app_icon: string
app_label: Record<string, string>
scope: string
}
export type OAuthAuthorizeResponse = {
code: string
}
export const useOAuthAppInfo = (client_id: string, redirect_uri: string, enabled = true) => {
return useQuery<OAuthAppInfo>({
queryKey: [NAME_SPACE, 'authAppInfo', client_id, redirect_uri],
queryFn: () => post<OAuthAppInfo>('/oauth/provider', { body: { client_id, redirect_uri } }, { silent: true }),
enabled: Boolean(enabled && client_id && redirect_uri),
})
}
export const useAuthorizeOAuthApp = () => {
return useMutation({
mutationKey: [NAME_SPACE, 'authorize'],
mutationFn: (payload: { client_id: string }) => post<OAuthAuthorizeResponse>('/oauth/provider/authorize', { body: payload }),
})
}