update text spliter

api/configs/packaging/__init__.py

@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
 
     CURRENT_VERSION: str = Field(
         description="Dify version",
-        default="0.12.0",
+        default="0.11.2",
     )
 
     COMMIT_SHA: str = Field(

api/core/app/task_pipeline/workflow_cycle_manage.py

@@ -3,7 +3,6 @@ import time
 from collections.abc import Mapping, Sequence
 from datetime import UTC, datetime
 from typing import Any, Optional, Union, cast
-from uuid import uuid4
 
 from sqlalchemy.orm import Session
 
@@ -81,38 +80,38 @@ class WorkflowCycleManage:
 
             inputs[f"sys.{key.value}"] = value
 
+        inputs = WorkflowEntry.handle_special_values(inputs)
+
         triggered_from = (
             WorkflowRunTriggeredFrom.DEBUGGING
             if self._application_generate_entity.invoke_from == InvokeFrom.DEBUGGER
             else WorkflowRunTriggeredFrom.APP_RUN
         )
 
-        # handle special values
-        inputs = WorkflowEntry.handle_special_values(inputs)
-
         # init workflow run
-        with Session(db.engine, expire_on_commit=False) as session:
-            workflow_run = WorkflowRun()
-            system_id = self._workflow_system_variables[SystemVariableKey.WORKFLOW_RUN_ID]
-            workflow_run.id = system_id or str(uuid4())
-            workflow_run.tenant_id = self._workflow.tenant_id
-            workflow_run.app_id = self._workflow.app_id
-            workflow_run.sequence_number = new_sequence_number
-            workflow_run.workflow_id = self._workflow.id
-            workflow_run.type = self._workflow.type
-            workflow_run.triggered_from = triggered_from.value
-            workflow_run.version = self._workflow.version
-            workflow_run.graph = self._workflow.graph
-            workflow_run.inputs = json.dumps(inputs)
-            workflow_run.status = WorkflowRunStatus.RUNNING
-            workflow_run.created_by_role = (
-                CreatedByRole.ACCOUNT if isinstance(self._user, Account) else CreatedByRole.END_USER
-            )
-            workflow_run.created_by = self._user.id
-            workflow_run.created_at = datetime.now(UTC).replace(tzinfo=None)
+        workflow_run = WorkflowRun()
+        workflow_run_id = self._workflow_system_variables[SystemVariableKey.WORKFLOW_RUN_ID]
+        if workflow_run_id:
+            workflow_run.id = workflow_run_id
+        workflow_run.tenant_id = self._workflow.tenant_id
+        workflow_run.app_id = self._workflow.app_id
+        workflow_run.sequence_number = new_sequence_number
+        workflow_run.workflow_id = self._workflow.id
+        workflow_run.type = self._workflow.type
+        workflow_run.triggered_from = triggered_from.value
+        workflow_run.version = self._workflow.version
+        workflow_run.graph = self._workflow.graph
+        workflow_run.inputs = json.dumps(inputs)
+        workflow_run.status = WorkflowRunStatus.RUNNING.value
+        workflow_run.created_by_role = (
+            CreatedByRole.ACCOUNT.value if isinstance(self._user, Account) else CreatedByRole.END_USER.value
+        )
+        workflow_run.created_by = self._user.id
 
-            session.add(workflow_run)
-            session.commit()
+        db.session.add(workflow_run)
+        db.session.commit()
+        db.session.refresh(workflow_run)
+        db.session.close()
 
         return workflow_run
 

api/core/indexing_runner.py

@@ -720,10 +720,8 @@ class IndexingRunner:
 
             tokens = 0
             if embedding_model_instance:
-                tokens += sum(
-                    embedding_model_instance.get_text_embedding_num_tokens([document.page_content])
-                    for document in chunk_documents
-                )
+                page_content_list = [document.page_content for document in chunk_documents]
+                tokens += sum(embedding_model_instance.get_text_embedding_num_tokens(page_content_list))
 
             # load index
             index_processor.load(dataset, chunk_documents, with_keywords=False)

api/core/model_manager.py

@@ -183,7 +183,7 @@ class ModelInstance:
             input_type=input_type,
         )
 
-    def get_text_embedding_num_tokens(self, texts: list[str]) -> int:
+    def get_text_embedding_num_tokens(self, texts: list[str]) -> list[int]:
         """
         Get number of tokens for text embedding
 

api/core/model_runtime/model_providers/azure_openai/_constant.py

@@ -779,7 +779,7 @@ LLM_BASE_MODELS = [
                     name="frequency_penalty",
                     **PARAMETER_RULE_TEMPLATE[DefaultParameterName.FREQUENCY_PENALTY],
                 ),
-                _get_max_tokens(default=512, min_val=1, max_val=16384),
+                _get_max_tokens(default=512, min_val=1, max_val=4096),
                 ParameterRule(
                     name="seed",
                     label=I18nObject(zh_Hans="种子", en_US="Seed"),

api/core/model_runtime/model_providers/gitee_ai/rerank/rerank.py

@@ -122,7 +122,7 @@ class GiteeAIRerankModel(RerankModel):
             label=I18nObject(en_US=model),
             model_type=ModelType.RERANK,
             fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
-            model_properties={ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size", 512))},
+            model_properties={ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size"))},
         )
 
         return entity

api/core/model_runtime/model_providers/gpustack/rerank/rerank.py

@@ -140,7 +140,7 @@ class GPUStackRerankModel(RerankModel):
             label=I18nObject(en_US=model),
             model_type=ModelType.RERANK,
             fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
-            model_properties={ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size", 512))},
+            model_properties={ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size"))},
         )
 
         return entity

api/core/model_runtime/model_providers/jina/rerank/rerank.py

@@ -128,7 +128,7 @@ class JinaRerankModel(RerankModel):
             label=I18nObject(en_US=model),
             model_type=ModelType.RERANK,
             fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
-            model_properties={ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size", 8000))},
+            model_properties={ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size"))},
         )
 
         return entity

api/core/model_runtime/model_providers/jina/text_embedding/text_embedding.py

@@ -193,7 +193,7 @@ class JinaTextEmbeddingModel(TextEmbeddingModel):
             label=I18nObject(en_US=model),
             model_type=ModelType.TEXT_EMBEDDING,
             fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
-            model_properties={ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size", 8000))},
+            model_properties={ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size"))},
         )
 
         return entity

api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py

@@ -139,7 +139,7 @@ class OllamaEmbeddingModel(TextEmbeddingModel):
             model_type=ModelType.TEXT_EMBEDDING,
             fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
             model_properties={
-                ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size", 512)),
+                ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size")),
                 ModelPropertyKey.MAX_CHUNKS: 1,
             },
             parameter_rules=[],

api/core/model_runtime/model_providers/openai_api_compatible/text_embedding/text_embedding.py

@@ -176,7 +176,7 @@ class OAICompatEmbeddingModel(_CommonOaiApiCompat, TextEmbeddingModel):
             model_type=ModelType.TEXT_EMBEDDING,
             fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
             model_properties={
-                ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size", 512)),
+                ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size")),
                 ModelPropertyKey.MAX_CHUNKS: 1,
             },
             parameter_rules=[],

api/core/model_runtime/model_providers/perfxcloud/text_embedding/text_embedding.py

@@ -182,7 +182,7 @@ class OAICompatEmbeddingModel(_CommonOaiApiCompat, TextEmbeddingModel):
             model_type=ModelType.TEXT_EMBEDDING,
             fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
             model_properties={
-                ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size", 512)),
+                ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size")),
                 ModelPropertyKey.MAX_CHUNKS: 1,
             },
             parameter_rules=[],

api/core/model_runtime/model_providers/vertex_ai/text_embedding/text_embedding.py

@@ -173,7 +173,7 @@ class VertexAiTextEmbeddingModel(_CommonVertexAi, TextEmbeddingModel):
             model_type=ModelType.TEXT_EMBEDDING,
             fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
             model_properties={
-                ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size", 512)),
+                ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size")),
                 ModelPropertyKey.MAX_CHUNKS: 1,
             },
             parameter_rules=[],

api/core/model_runtime/model_providers/voyage/text_embedding/text_embedding.py

@@ -166,7 +166,7 @@ class VoyageTextEmbeddingModel(TextEmbeddingModel):
             label=I18nObject(en_US=model),
             model_type=ModelType.TEXT_EMBEDDING,
             fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
-            model_properties={ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size", 512))},
+            model_properties={ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size"))},
         )
 
         return entity

api/core/model_runtime/model_providers/x/llm/grok-beta.yaml

@@ -1,12 +1,9 @@
 model: grok-beta
 label:
-  en_US: Grok Beta
+  en_US: Grok beta
 model_type: llm
 features:
-  - agent-thought
-  - tool-call
   - multi-tool-call
-  - stream-tool-call
 model_properties:
   mode: chat
   context_size: 131072

api/core/model_runtime/model_providers/x/llm/grok-vision-beta.yaml

@@ -1,64 +0,0 @@
-model: grok-vision-beta
-label:
-  en_US: Grok Vision Beta
-model_type: llm
-features:
-  - agent-thought
-  - vision
-model_properties:
-  mode: chat
-  context_size: 8192
-parameter_rules:
-  - name: temperature
-    label:
-      en_US: "Temperature"
-      zh_Hans: "采样温度"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 2.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The randomness of the sampling temperature control output. The temperature value is within the range of [0.0, 1.0]. The higher the value, the more random and creative the output; the lower the value, the more stable it is. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样温度控制输出的随机性。温度值在 [0.0, 1.0] 范围内，值越高，输出越随机和创造性；值越低，输出越稳定。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: top_p
-    label:
-      en_US: "Top P"
-      zh_Hans: "Top P"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The value range of the sampling method is [0.0, 1.0]. The top_p value determines that the model selects tokens from the top p% of candidate words with the highest probability; when top_p is 0, this parameter is invalid. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样方法的取值范围为 [0.0,1.0]。top_p 值确定模型从概率最高的前p%的候选词中选取 tokens；当 top_p 为 0 时，此参数无效。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: frequency_penalty
-    use_template: frequency_penalty
-    label:
-      en_US: "Frequency Penalty"
-      zh_Hans: "频率惩罚"
-    type: float
-    default: 0
-    min: 0
-    max: 2.0
-    precision: 1
-    required: false
-    help:
-      en_US: "Number between 0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim."
-      zh_Hans: "介于0和2.0之间的数字。正值会根据新标记在文本中迄今为止的现有频率来惩罚它们，从而降低模型一字不差地重复同一句话的可能性。"
-
-  - name: user
-    use_template: text
-    label:
-      en_US: "User"
-      zh_Hans: "用户"
-    type: string
-    required: false
-    help:
-      en_US: "Used to track and differentiate conversation requests from different users."
-      zh_Hans: "用于追踪和区分不同用户的对话请求。"

api/core/model_runtime/model_providers/x/llm/llm.py

@@ -35,5 +35,3 @@ class XAILargeLanguageModel(OAIAPICompatLargeLanguageModel):
         credentials["endpoint_url"] = str(URL(credentials["endpoint_url"])) or "https://api.x.ai/v1"
         credentials["mode"] = LLMMode.CHAT.value
         credentials["function_calling_type"] = "tool_call"
-        credentials["stream_function_calling"] = "support"
-        credentials["vision_support"] = "support"

api/core/model_runtime/model_providers/zhipuai/text_embedding/text_embedding.py

@@ -105,6 +105,17 @@ class ZhipuAITextEmbeddingModel(_CommonZhipuaiAI, TextEmbeddingModel):
 
         return [list(map(float, e)) for e in embeddings], embedding_used_tokens
 
+    def embed_query(self, text: str) -> list[float]:
+        """Call out to ZhipuAI's embedding endpoint.
+
+        Args:
+            text: The text to embed.
+
+        Returns:
+            Embeddings for the text.
+        """
+        return self.embed_documents([text])[0]
+
     def _calc_response_usage(self, model: str, credentials: dict, tokens: int) -> EmbeddingUsage:
         """
         Calculate response usage

api/core/rag/docstore/dataset_docstore.py

@@ -78,8 +78,13 @@ class DatasetDocumentStore:
                 model_type=ModelType.TEXT_EMBEDDING,
                 model=self._dataset.embedding_model,
             )
+        if embedding_model:
+            page_content_list = [doc.page_content for doc in docs]
+            tokens_list = embedding_model.get_text_embedding_num_tokens(page_content_list)
+        else:
+            tokens_list = [0] * len(docs)
 
-        for doc in docs:
+        for doc, tokens in zip(docs, tokens_list):
             if not isinstance(doc, Document):
                 raise ValueError("doc must be a Document")
 
@@ -91,12 +96,6 @@ class DatasetDocumentStore:
                     f"doc_id {doc.metadata['doc_id']} already exists. Set allow_update to True to overwrite."
                 )
 
-            # calc embedding use tokens
-            if embedding_model:
-                tokens = embedding_model.get_text_embedding_num_tokens(texts=[doc.page_content])
-            else:
-                tokens = 0
-
             if not segment_document:
                 max_position += 1
 

api/core/rag/splitter/fixed_text_splitter.py

@@ -65,8 +65,9 @@ class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter)
             chunks = [text]
 
         final_chunks = []
-        for chunk in chunks:
-            if self._length_function(chunk) > self._chunk_size:
+        chunks_lengths = self._length_function(chunks)
+        for chunk, chunk_length in zip(chunks, chunks_lengths):
+            if chunk_length > self._chunk_size:
                 final_chunks.extend(self.recursive_split_text(chunk))
             else:
                 final_chunks.append(chunk)
@@ -93,8 +94,8 @@ class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter)
         # Now go merging things, recursively splitting longer texts.
         _good_splits = []
         _good_splits_lengths = []  # cache the lengths of the splits
-        for s in splits:
-            s_len = self._length_function(s)
+        s_lens = self._length_function(splits)
+        for s, s_len in zip(splits, s_lens):
             if s_len < self._chunk_size:
                 _good_splits.append(s)
                 _good_splits_lengths.append(s_len)

api/core/rag/splitter/text_splitter.py

@@ -45,7 +45,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):
         self,
         chunk_size: int = 4000,
         chunk_overlap: int = 200,
-        length_function: Callable[[str], int] = len,
+        length_function: Callable[[str], [int]] = len,
         keep_separator: bool = False,
         add_start_index: bool = False,
     ) -> None:
@@ -224,8 +224,8 @@ class CharacterTextSplitter(TextSplitter):
         splits = _split_text_with_regex(text, self._separator, self._keep_separator)
         _separator = "" if self._keep_separator else self._separator
         _good_splits_lengths = []  # cache the lengths of the splits
-        for split in splits:
-            _good_splits_lengths.append(self._length_function(split))
+        if splits:
+            _good_splits_lengths.extend(self._length_function(splits))
         return self._merge_splits(splits, _separator, _good_splits_lengths)
 
 
@@ -478,9 +478,8 @@ class RecursiveCharacterTextSplitter(TextSplitter):
         _good_splits = []
         _good_splits_lengths = []  # cache the lengths of the splits
         _separator = "" if self._keep_separator else separator
-
-        for s in splits:
-            s_len = self._length_function(s)
+        s_lens = self._length_function(splits)
+        for s, s_len in zip(splits, s_lens):
             if s_len < self._chunk_size:
                 _good_splits.append(s)
                 _good_splits_lengths.append(s_len)

api/core/tools/provider/builtin/chart/chart.py

@@ -1,4 +1,3 @@
-import matplotlib
 import matplotlib.pyplot as plt
 from matplotlib.font_manager import FontProperties, fontManager
 
@@ -6,7 +5,7 @@ from core.tools.provider.builtin_tool_provider import BuiltinToolProviderControl
 
 
 def set_chinese_font():
-    to_find_fonts = [
+    font_list = [
         "PingFang SC",
         "SimHei",
         "Microsoft YaHei",
@@ -16,16 +15,16 @@ def set_chinese_font():
         "Noto Sans CJK SC",
         "Noto Sans CJK JP",
     ]
-    installed_fonts = frozenset(fontInfo.name for fontInfo in fontManager.ttflist)
-    for font in to_find_fonts:
-        if font in installed_fonts:
-            return FontProperties(font)
+
+    for font in font_list:
+        if font in fontManager.ttflist:
+            chinese_font = FontProperties(font)
+            if chinese_font.get_name() == font:
+                return chinese_font
 
     return FontProperties()
 
 
-# use non-interactive backend to prevent `RuntimeError: main thread is not in main loop`
-matplotlib.use("Agg")
 # use a business theme
 plt.style.use("seaborn-v0_8-darkgrid")
 plt.rcParams["axes.unicode_minus"] = False

api/models/workflow.py

@@ -1,7 +1,7 @@
 import json
 from collections.abc import Mapping, Sequence
 from datetime import UTC, datetime
-from enum import Enum, StrEnum
+from enum import Enum
 from typing import Any, Optional, Union
 
 import sqlalchemy as sa
@@ -314,7 +314,7 @@ class Workflow(db.Model):
         )
 
 
-class WorkflowRunStatus(StrEnum):
+class WorkflowRunStatus(Enum):
     """
     Workflow Run Status Enum
     """
@@ -393,13 +393,13 @@ class WorkflowRun(db.Model):
     version = db.Column(db.String(255), nullable=False)
     graph = db.Column(db.Text)
     inputs = db.Column(db.Text)
-    status = db.Column(db.String(255), nullable=False)  # running, succeeded, failed, stopped
-    outputs: Mapped[str] = mapped_column(sa.Text, default="{}")
+    status = db.Column(db.String(255), nullable=False)
+    outputs: Mapped[str] = db.Column(db.Text)
     error = db.Column(db.Text)
     elapsed_time = db.Column(db.Float, nullable=False, server_default=db.text("0"))
     total_tokens = db.Column(db.Integer, nullable=False, server_default=db.text("0"))
     total_steps = db.Column(db.Integer, server_default=db.text("0"))
-    created_by_role = db.Column(db.String(255), nullable=False)  # account, end_user
+    created_by_role = db.Column(db.String(255), nullable=False)
     created_by = db.Column(StringUUID, nullable=False)
     created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))
     finished_at = db.Column(db.DateTime)

api/services/conversation_service.py

@@ -1,4 +1,3 @@
-from collections.abc import Callable
 from datetime import UTC, datetime
 from typing import Optional, Union
 
@@ -75,14 +74,14 @@ class ConversationService:
         return InfiniteScrollPagination(data=conversations, limit=limit, has_more=has_more)
 
     @classmethod
-    def _get_sort_params(cls, sort_by: str):
+    def _get_sort_params(cls, sort_by: str) -> tuple[str, callable]:
         if sort_by.startswith("-"):
             return sort_by[1:], desc
         return sort_by, asc
 
     @classmethod
     def _build_filter_condition(
-        cls, sort_field: str, sort_direction: Callable, reference_conversation: Conversation, is_next_page: bool = False
+        cls, sort_field: str, sort_direction: callable, reference_conversation: Conversation, is_next_page: bool = False
     ):
         field_value = getattr(reference_conversation, sort_field)
         if (sort_direction == desc and not is_next_page) or (sort_direction == asc and is_next_page):
@@ -161,5 +160,5 @@ class ConversationService:
         conversation = cls.get_conversation(app_model, conversation_id, user)
 
         conversation.is_deleted = True
-        conversation.updated_at = datetime.now(UTC).replace(tzinfo=None)
+        conversation.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
         db.session.commit()

api/services/dataset_service.py

@@ -1390,7 +1390,7 @@ class SegmentService:
                 model=dataset.embedding_model,
             )
             # calc embedding use tokens
-            tokens = embedding_model.get_text_embedding_num_tokens(texts=[content])
+            tokens = embedding_model.get_text_embedding_num_tokens(texts=[content])[0]
         lock_name = "add_segment_lock_document_id_{}".format(document.id)
         with redis_client.lock(lock_name, timeout=600):
             max_position = (
@@ -1467,9 +1467,11 @@ class SegmentService:
                 if dataset.indexing_technique == "high_quality" and embedding_model:
                     # calc embedding use tokens
                     if document.doc_form == "qa_model":
-                        tokens = embedding_model.get_text_embedding_num_tokens(texts=[content + segment_item["answer"]])
+                        tokens = embedding_model.get_text_embedding_num_tokens(
+                            texts=[content + segment_item["answer"]]
+                        )[0]
                     else:
-                        tokens = embedding_model.get_text_embedding_num_tokens(texts=[content])
+                        tokens = embedding_model.get_text_embedding_num_tokens(texts=[content])[0]
                 segment_document = DocumentSegment(
                     tenant_id=current_user.current_tenant_id,
                     dataset_id=document.dataset_id,
@@ -1577,9 +1579,9 @@ class SegmentService:
 
                     # calc embedding use tokens
                     if document.doc_form == "qa_model":
-                        tokens = embedding_model.get_text_embedding_num_tokens(texts=[content + segment.answer])
+                        tokens = embedding_model.get_text_embedding_num_tokens(texts=[content + segment.answer])[0]
                     else:
-                        tokens = embedding_model.get_text_embedding_num_tokens(texts=[content])
+                        tokens = embedding_model.get_text_embedding_num_tokens(texts=[content])[0]
                 segment.content = content
                 segment.index_node_hash = segment_hash
                 segment.word_count = len(content)

api/tasks/batch_create_segment_to_index_task.py

@@ -58,12 +58,16 @@ def batch_create_segment_to_index_task(
                 model=dataset.embedding_model,
             )
         word_count_change = 0
-        for segment in content:
+        if embedding_model:
+            tokens_list = embedding_model.get_text_embedding_num_tokens(
+                texts=[segment["content"] for segment in content]
+            )
+        else:
+            tokens_list = [0] * len(content)
+        for segment, tokens in zip(content, tokens_list):
             content = segment["content"]
             doc_id = str(uuid.uuid4())
             segment_hash = helper.generate_text_hash(content)
-            # calc embedding use tokens
-            tokens = embedding_model.get_text_embedding_num_tokens(texts=[content]) if embedding_model else 0
             max_position = (
                 db.session.query(func.max(DocumentSegment.position))
                 .filter(DocumentSegment.document_id == dataset_document.id)

docker-legacy/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3'
 services:
   # API service
   api:
-    image: langgenius/dify-api:0.12.0
+    image: langgenius/dify-api:0.11.2
     restart: always
     environment:
       # Startup mode, 'api' starts the API server.
@@ -227,7 +227,7 @@ services:
   # worker service
   # The Celery worker for processing the queue.
   worker:
-    image: langgenius/dify-api:0.12.0
+    image: langgenius/dify-api:0.11.2
     restart: always
     environment:
       CONSOLE_WEB_URL: ''
@@ -397,7 +397,7 @@ services:
 
   # Frontend web application.
   web:
-    image: langgenius/dify-web:0.12.0
+    image: langgenius/dify-web:0.11.2
     restart: always
     environment:
       # The base URL of console application api server, refers to the Console base URL of WEB service if console domain is

docker/docker-compose.yaml

@@ -291,7 +291,7 @@ x-shared-env: &shared-api-worker-env
 services:
   # API service
   api:
-    image: langgenius/dify-api:0.12.0
+    image: langgenius/dify-api:0.11.2
     restart: always
     environment:
       # Use the shared environment variables.
@@ -311,7 +311,7 @@ services:
   # worker service
   # The Celery worker for processing the queue.
   worker:
-    image: langgenius/dify-api:0.12.0
+    image: langgenius/dify-api:0.11.2
     restart: always
     environment:
       # Use the shared environment variables.
@@ -330,7 +330,7 @@ services:
 
   # Frontend web application.
   web:
-    image: langgenius/dify-web:0.12.0
+    image: langgenius/dify-web:0.11.2
     restart: always
     environment:
       CONSOLE_API_URL: ${CONSOLE_API_URL:-}

web/app/components/base/app-icon-picker/index.tsx

@@ -69,7 +69,7 @@ const AppIconPicker: FC<AppIconPickerProps> = ({
     },
   })
 
-  const [imageCropInfo, setImageCropInfo] = useState<{ tempUrl: string; croppedAreaPixels: Area; fileName: string; file?: File }>()
+  const [imageCropInfo, setImageCropInfo] = useState<{ tempUrl: string; croppedAreaPixels: Area; fileName: string }>()
   const handleImageCropped = async (tempUrl: string, croppedAreaPixels: Area, fileName: string) => {
     setImageCropInfo({ tempUrl, croppedAreaPixels, fileName })
   }
@@ -93,15 +93,13 @@ const AppIconPicker: FC<AppIconPickerProps> = ({
       if (!imageCropInfo && !uploadImageInfo)
         return
       setUploading(true)
-      if (imageCropInfo?.file) {
+      if (imageCropInfo.file) {
         handleLocalFileUpload(imageCropInfo.file)
         return
       }
-      if (imageCropInfo) {
-        const blob = await getCroppedImg(imageCropInfo.tempUrl, imageCropInfo.croppedAreaPixels, imageCropInfo.fileName)
-        const file = new File([blob], imageCropInfo.fileName, { type: blob.type })
-        handleLocalFileUpload(file)
-      }
+      const blob = await getCroppedImg(imageCropInfo.tempUrl, imageCropInfo.croppedAreaPixels, imageCropInfo.fileName)
+      const file = new File([blob], imageCropInfo.fileName, { type: blob.type })
+      handleLocalFileUpload(file)
     }
   }
 

web/app/components/base/app-icon-picker/utils.ts

@@ -116,15 +116,11 @@ export default async function getCroppedImg(
   })
 }
 
-export function checkIsAnimatedImage(file: Blob) {
+export function checkIsAnimatedImage(file) {
   return new Promise((resolve, reject) => {
     const fileReader = new FileReader()
 
     fileReader.onload = function (e) {
-      if (!e.target || !(e.target.result instanceof ArrayBuffer)) {
-        reject(new Error('File reading error'))
-        return
-      }
       const arr = new Uint8Array(e.target.result)
 
       // Check file extension
@@ -152,7 +148,7 @@ export function checkIsAnimatedImage(file: Blob) {
 }
 
 // Function to check for WebP signature
-function isWebP(arr: Uint8Array): boolean {
+function isWebP(arr) {
   return (
     arr[0] === 0x52 && arr[1] === 0x49 && arr[2] === 0x46 && arr[3] === 0x46
     && arr[8] === 0x57 && arr[9] === 0x45 && arr[10] === 0x42 && arr[11] === 0x50
@@ -160,7 +156,7 @@ function isWebP(arr: Uint8Array): boolean {
 }
 
 // Function to check if the WebP is animated (contains ANIM chunk)
-function checkWebPAnimation(arr: string | any[] | Uint8Array) {
+function checkWebPAnimation(arr) {
   // Search for the ANIM chunk in WebP to determine if it's animated
   for (let i = 12; i < arr.length - 4; i++) {
     if (arr[i] === 0x41 && arr[i + 1] === 0x4E && arr[i + 2] === 0x49 && arr[i + 3] === 0x4D)

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dify-web",
-  "version": "0.12.0",
+  "version": "0.11.2",
   "private": true,
   "engines": {
     "node": ">=18.17.0"