bump version to 0.4.9 (#2103)

Co-authored-by: jyong <jyong@dify.ai>

.github/workflows/style.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Setup NodeJS
         uses: actions/setup-node@v4
         with:
-          node-version: 18
+          node-version: 20
           cache: yarn
           cache-dependency-path: ./web/package.json
 

CONTRIBUTING.md

@@ -12,7 +12,7 @@ In terms of licensing, please take a minute to read our short [License and Contr
 
 ### Feature requests:
 
-* If you're opening a new feature request, we'd like you to explain what the proposed feature achieves, and include as much context as possible.
+* If you're opening a new feature request, we'd like you to explain what the proposed feature achieves, and include as much context as possible. [@perzeusss](https://github.com/perzeuss) has made a solid [Feature Request Copilot](https://udify.app/chat/MK2kVSnw1gakVwMX) that helps you draft out your needs. Feel free to give it a try.
 
 * If you want to pick one up from the existing issues, simply drop a comment below it saying so.
 

api/.env.example

@@ -15,7 +15,6 @@ CONSOLE_WEB_URL=http://127.0.0.1:3000
 SERVICE_API_URL=http://127.0.0.1:5001
 
 # Web APP base URL
-APP_API_URL=http://127.0.0.1:5001
 APP_WEB_URL=http://127.0.0.1:3000
 
 # Files URL

api/Dockerfile

@@ -13,17 +13,20 @@ RUN pip install --prefix=/pkg -r requirements.txt
 # build stage
 FROM python:3.10-slim AS builder
 
+
 ENV FLASK_APP app.py
 ENV EDITION SELF_HOSTED
 ENV DEPLOY_ENV PRODUCTION
 ENV CONSOLE_API_URL http://127.0.0.1:5001
 ENV CONSOLE_WEB_URL http://127.0.0.1:3000
 ENV SERVICE_API_URL http://127.0.0.1:5001
-ENV APP_API_URL http://127.0.0.1:5001
 ENV APP_WEB_URL http://127.0.0.1:3000
 
 EXPOSE 5001
 
+# set timezone
+ENV TZ UTC
+
 WORKDIR /app/api
 
 RUN apt-get update \

api/config.py

@@ -22,7 +22,6 @@ DEFAULTS = {
     'CONSOLE_API_URL': 'https://cloud.dify.ai',
     'SERVICE_API_URL': 'https://api.dify.ai',
     'APP_WEB_URL': 'https://udify.app',
-    'APP_API_URL': 'https://udify.app',
     'FILES_URL': '',
     'STORAGE_TYPE': 'local',
     'STORAGE_LOCAL_PATH': 'storage',
@@ -94,7 +93,7 @@ class Config:
         # ------------------------
         # General Configurations.
         # ------------------------
-        self.CURRENT_VERSION = "0.4.8"
+        self.CURRENT_VERSION = "0.4.9"
         self.COMMIT_SHA = get_env('COMMIT_SHA')
         self.EDITION = "SELF_HOSTED"
         self.DEPLOY_ENV = get_env('DEPLOY_ENV')
@@ -103,35 +102,25 @@ class Config:
 
         # The backend URL prefix of the console API.
         # used to concatenate the login authorization callback or notion integration callback.
-        self.CONSOLE_API_URL = get_env('CONSOLE_URL') if get_env('CONSOLE_URL') else get_env('CONSOLE_API_URL')
+        self.CONSOLE_API_URL = get_env('CONSOLE_API_URL')
 
         # The front-end URL prefix of the console web.
         # used to concatenate some front-end addresses and for CORS configuration use.
-        self.CONSOLE_WEB_URL = get_env('CONSOLE_URL') if get_env('CONSOLE_URL') else get_env('CONSOLE_WEB_URL')
-
-        # WebApp API backend Url prefix.
-        # used to declare the back-end URL for the front-end API.
-        self.APP_API_URL = get_env('APP_URL') if get_env('APP_URL') else get_env('APP_API_URL')
+        self.CONSOLE_WEB_URL = get_env('CONSOLE_WEB_URL')
 
         # WebApp Url prefix.
         # used to display WebAPP API Base Url to the front-end.
-        self.APP_WEB_URL = get_env('APP_URL') if get_env('APP_URL') else get_env('APP_WEB_URL')
+        self.APP_WEB_URL = get_env('APP_WEB_URL')
 
         # Service API Url prefix.
         # used to display Service API Base Url to the front-end.
-        self.SERVICE_API_URL = get_env('API_URL') if get_env('API_URL') else get_env('SERVICE_API_URL')
+        self.SERVICE_API_URL = get_env('SERVICE_API_URL')
 
         # File preview or download Url prefix.
         # used to display File preview or download Url to the front-end or as Multi-model inputs;
         # Url is signed and has expiration time.
         self.FILES_URL = get_env('FILES_URL') if get_env('FILES_URL') else self.CONSOLE_API_URL
 
-        # Fallback Url prefix.
-        # Will be deprecated in the future.
-        self.CONSOLE_URL = get_env('CONSOLE_URL')
-        self.API_URL = get_env('API_URL')
-        self.APP_URL = get_env('APP_URL')
-
         # Your App secret key will be used for securely signing the session cookie
         # Make sure you are changing this key for your deployment with a strong key.
         # You can generate a strong key using `openssl rand -base64 42`.

api/controllers/console/datasets/datasets.py

@@ -19,7 +19,7 @@ from flask import current_app, request
 from flask_login import current_user
 from flask_restful import Resource, marshal, marshal_with, reqparse
 from libs.login import login_required
-from models.dataset import Document, DocumentSegment
+from models.dataset import Dataset, Document, DocumentSegment
 from models.model import ApiToken, UploadFile
 from services.dataset_service import DatasetService, DocumentService
 from werkzeug.exceptions import Forbidden, NotFound
@@ -97,7 +97,8 @@ class DatasetListApi(Resource):
                             help='type is required. Name must be between 1 to 40 characters.',
                             type=_validate_name)
         parser.add_argument('indexing_technique', type=str, location='json',
-                            choices=('high_quality', 'economy'),
+                            choices=Dataset.INDEXING_TECHNIQUE_LIST,
+                            nullable=True,
                             help='Invalid indexing technique.')
         args = parser.parse_args()
 
@@ -177,8 +178,9 @@ class DatasetApi(Resource):
                             location='json', store_missing=False,
                             type=_validate_description_length)
         parser.add_argument('indexing_technique', type=str, location='json',
-                            choices=('high_quality', 'economy'),
-                            help='Invalid indexing technique.')
+                    choices=Dataset.INDEXING_TECHNIQUE_LIST,
+                    nullable=True,
+                    help='Invalid indexing technique.')
         parser.add_argument('permission', type=str, location='json', choices=(
             'only_me', 'all_team_members'), help='Invalid permission.')
         parser.add_argument('retrieval_model', type=dict, location='json', help='Invalid retrieval model.')
@@ -256,7 +258,9 @@ class DatasetIndexingEstimateApi(Resource):
         parser = reqparse.RequestParser()
         parser.add_argument('info_list', type=dict, required=True, nullable=True, location='json')
         parser.add_argument('process_rule', type=dict, required=True, nullable=True, location='json')
-        parser.add_argument('indexing_technique', type=str, required=True, nullable=True, location='json')
+        parser.add_argument('indexing_technique', type=str, required=True, 
+                            choices=Dataset.INDEXING_TECHNIQUE_LIST,
+                            nullable=True, location='json')
         parser.add_argument('doc_form', type=str, default='text_model', required=False, nullable=False, location='json')
         parser.add_argument('dataset_id', type=str, required=False, nullable=False, location='json')
         parser.add_argument('doc_language', type=str, default='English', required=False, nullable=False,

api/controllers/service_api/dataset/dataset.py

@@ -1,3 +1,4 @@
+from models.dataset import Dataset
 import services.dataset_service
 from controllers.service_api import api
 from controllers.service_api.dataset.error import DatasetNameDuplicateError
@@ -68,7 +69,7 @@ class DatasetApi(DatasetApiResource):
                             help='type is required. Name must be between 1 to 40 characters.',
                             type=_validate_name)
         parser.add_argument('indexing_technique', type=str, location='json',
-                            choices=('high_quality', 'economy'),
+                            choices=Dataset.INDEXING_TECHNIQUE_LIST,
                             help='Invalid indexing technique.')
         args = parser.parse_args()
 

api/controllers/service_api/wraps.py

@@ -75,7 +75,7 @@ def validate_dataset_token(view=None):
             tenant_account_join = db.session.query(Tenant, TenantAccountJoin) \
                 .filter(Tenant.id == api_token.tenant_id) \
                 .filter(TenantAccountJoin.tenant_id == Tenant.id) \
-                .filter(TenantAccountJoin.role == 'owner') \
+                .filter(TenantAccountJoin.role.in_(['owner', 'admin'])) \
                 .one_or_none()
             if tenant_account_join:
                 tenant, ta = tenant_account_join

api/core/embedding/cached_embedding.py

@@ -1,10 +1,16 @@
+import base64
+import json
 import logging
-from typing import List, Optional
+from typing import List, Optional, cast
 
 import numpy as np
 from core.model_manager import ModelInstance
+from core.model_runtime.entities.model_entities import ModelPropertyKey
+from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
 from extensions.ext_database import db
 from langchain.embeddings.base import Embeddings
+
+from extensions.ext_redis import redis_client
 from libs import helper
 from models.dataset import Embedding
 from sqlalchemy.exc import IntegrityError
@@ -18,47 +24,33 @@ class CacheEmbedding(Embeddings):
         self._user = user
 
     def embed_documents(self, texts: List[str]) -> List[List[float]]:
-        """Embed search docs."""
-        # use doc embedding cache or store if not exists
-        text_embeddings = [None for _ in range(len(texts))]
-        embedding_queue_indices = []
-        for i, text in enumerate(texts):
-            hash = helper.generate_text_hash(text)
-            embedding = db.session.query(Embedding).filter_by(model_name=self._model_instance.model, hash=hash).first()
-            if embedding:
-                text_embeddings[i] = embedding.get_embedding()
-            else:
-                embedding_queue_indices.append(i)
+        """Embed search docs in batches of 10."""
+        text_embeddings = []
+        try:
+            model_type_instance = cast(TextEmbeddingModel, self._model_instance.model_type_instance)
+            model_schema = model_type_instance.get_model_schema(self._model_instance.model, self._model_instance.credentials)
+            max_chunks = model_schema.model_properties[ModelPropertyKey.MAX_CHUNKS] \
+                if model_schema and ModelPropertyKey.MAX_CHUNKS in model_schema.model_properties else 1
+            for i in range(0, len(texts), max_chunks):
+                batch_texts = texts[i:i + max_chunks]
 
-        if embedding_queue_indices:
-            try:
                 embedding_result = self._model_instance.invoke_text_embedding(
-                    texts=[texts[i] for i in embedding_queue_indices],
+                    texts=batch_texts,
                     user=self._user
                 )
 
-                embedding_results = embedding_result.embeddings
-            except Exception as ex:
-                logger.error('Failed to embed documents: ', ex)
-                raise ex
+                for vector in embedding_result.embeddings:
+                    try:
+                        normalized_embedding = (vector / np.linalg.norm(vector)).tolist()
+                        text_embeddings.append(normalized_embedding)
+                    except IntegrityError:
+                        db.session.rollback()
+                    except Exception as e:
+                        logging.exception('Failed to add embedding to redis')
 
-            for i, indice in enumerate(embedding_queue_indices):
-                hash = helper.generate_text_hash(texts[indice])
-
-                try:
-                    embedding = Embedding(model_name=self._model_instance.model, hash=hash)
-                    vector = embedding_results[i]
-                    normalized_embedding = (vector / np.linalg.norm(vector)).tolist()
-                    text_embeddings[indice] = normalized_embedding
-                    embedding.set_embedding(normalized_embedding)
-                    db.session.add(embedding)
-                    db.session.commit()
-                except IntegrityError:
-                    db.session.rollback()
-                    continue
-                except:
-                    logging.exception('Failed to add embedding to db')
-                    continue
+        except Exception as ex:
+            logger.error('Failed to embed documents: ', ex)
+            raise ex
 
         return text_embeddings
 
@@ -66,9 +58,12 @@ class CacheEmbedding(Embeddings):
         """Embed query text."""
         # use doc embedding cache or store if not exists
         hash = helper.generate_text_hash(text)
-        embedding = db.session.query(Embedding).filter_by(model_name=self._model_instance.model, hash=hash).first()
+        embedding_cache_key = f'{self._model_instance.provider}_{self._model_instance.model}_{hash}'
+        embedding = redis_client.get(embedding_cache_key)
         if embedding:
-            return embedding.get_embedding()
+            redis_client.expire(embedding_cache_key, 600)
+            return list(np.frombuffer(base64.b64decode(embedding), dtype="float"))
+
 
         try:
             embedding_result = self._model_instance.invoke_text_embedding(
@@ -82,13 +77,18 @@ class CacheEmbedding(Embeddings):
             raise ex
 
         try:
-            embedding = Embedding(model_name=self._model_instance.model, hash=hash)
-            embedding.set_embedding(embedding_results)
-            db.session.add(embedding)
-            db.session.commit()
+            # encode embedding to base64
+            embedding_vector = np.array(embedding_results)
+            vector_bytes = embedding_vector.tobytes()
+            # Transform to Base64
+            encoded_vector = base64.b64encode(vector_bytes)
+            # Transform to string
+            encoded_str = encoded_vector.decode("utf-8")
+            redis_client.setex(embedding_cache_key, 600, encoded_str)
+
         except IntegrityError:
             db.session.rollback()
         except:
-            logging.exception('Failed to add embedding to db')
+            logging.exception('Failed to add embedding to redis')
 
         return embedding_results

api/core/indexing_runner.py

@@ -274,6 +274,8 @@ class IndexingRunner:
         tokens = 0
         preview_texts = []
         total_segments = 0
+        total_price = 0
+        currency = 'USD'
         for file_detail in file_details:
 
             processing_rule = DatasetProcessRule(
@@ -344,11 +346,13 @@ class IndexingRunner:
                 price_type=PriceType.INPUT,
                 tokens=tokens
             )
+            total_price = '{:f}'.format(embedding_price_info.total_amount)
+            currency = embedding_price_info.currency
         return {
             "total_segments": total_segments,
             "tokens": tokens,
-            "total_price": '{:f}'.format(embedding_price_info.total_amount) if embedding_model_instance else 0,
-            "currency": embedding_price_info.currency if embedding_model_instance else 'USD',
+            "total_price": total_price,
+            "currency": currency,
             "preview": preview_texts
         }
 
@@ -388,6 +392,8 @@ class IndexingRunner:
         tokens = 0
         preview_texts = []
         total_segments = 0
+        total_price = 0
+        currency = 'USD'
         for notion_info in notion_info_list:
             workspace_id = notion_info['workspace_id']
             data_source_binding = DataSourceBinding.query.filter(
@@ -470,20 +476,22 @@ class IndexingRunner:
                     "qa_preview": document_qa_list,
                     "preview": preview_texts
                 }
-
-        embedding_model_type_instance = embedding_model_instance.model_type_instance
-        embedding_model_type_instance = cast(TextEmbeddingModel, embedding_model_type_instance)
-        embedding_price_info = embedding_model_type_instance.get_price(
-            model=embedding_model_instance.model,
-            credentials=embedding_model_instance.credentials,
-            price_type=PriceType.INPUT,
-            tokens=tokens
-        )
+        if embedding_model_instance:
+            embedding_model_type_instance = embedding_model_instance.model_type_instance
+            embedding_model_type_instance = cast(TextEmbeddingModel, embedding_model_type_instance)
+            embedding_price_info = embedding_model_type_instance.get_price(
+                model=embedding_model_instance.model,
+                credentials=embedding_model_instance.credentials,
+                price_type=PriceType.INPUT,
+                tokens=tokens
+            )
+            total_price = '{:f}'.format(embedding_price_info.total_amount)
+            currency = embedding_price_info.currency
         return {
             "total_segments": total_segments,
             "tokens": tokens,
-            "total_price": '{:f}'.format(embedding_price_info.total_amount) if embedding_model_instance else 0,
-            "currency": embedding_price_info.currency if embedding_model_instance else 'USD',
+            "total_price": total_price,
+            "currency": currency,
             "preview": preview_texts
         }
 

api/core/model_runtime/model_providers/jina/text_embedding/jina-embeddings-v2-base-de.yaml

@@ -0,0 +1,9 @@
+model: jina-embeddings-v2-base-de
+model_type: text-embedding
+model_properties:
+  context_size: 8192
+  max_chunks: 2048
+pricing:
+  input: '0.001'
+  unit: '0.001'
+  currency: USD

api/core/model_runtime/model_providers/jina/text_embedding/text_embedding.py

@@ -17,7 +17,7 @@ class JinaTextEmbeddingModel(TextEmbeddingModel):
     Model class for Jina text embedding model.
     """
     api_base: str = 'https://api.jina.ai/v1/embeddings'
-    models: list[str] = ['jina-embeddings-v2-base-en', 'jina-embeddings-v2-small-en', 'jina-embeddings-v2-base-zh']
+    models: list[str] = ['jina-embeddings-v2-base-en', 'jina-embeddings-v2-small-en', 'jina-embeddings-v2-base-zh', 'jina-embeddings-v2-base-de']
 
     def _invoke(self, model: str, credentials: dict,
                 texts: list[str], user: Optional[str] = None) \

api/models/dataset.py

@@ -17,7 +17,7 @@ class Dataset(db.Model):
         db.Index('retrieval_model_idx', "retrieval_model", postgresql_using='gin')
     )
 
-    INDEXING_TECHNIQUE_LIST = ['high_quality', 'economy']
+    INDEXING_TECHNIQUE_LIST = ['high_quality', 'economy', None]
 
     id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
     tenant_id = db.Column(UUID, nullable=False)

docker/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.1'
 services:
   # API service
   api:
-    image: langgenius/dify-api:0.4.8
+    image: langgenius/dify-api:0.4.9
     restart: always
     environment:
       # Startup mode, 'api' starts the API server.
@@ -23,10 +23,6 @@ services:
       # different from console domain.
       # example: http://api.dify.ai
       SERVICE_API_URL: ''
-      # The URL prefix for Web APP api server, refers to the Web App base URL of WEB service if web app domain is different from
-      # console or api domain.
-      # example: http://udify.app
-      APP_API_URL: ''
       # The URL prefix for Web APP frontend, refers to the Web App base URL of WEB service if web app domain is different from
       # console or api domain.
       # example: http://udify.app
@@ -131,7 +127,7 @@ services:
   # worker service
   # The Celery worker for processing the queue.
   worker:
-    image: langgenius/dify-api:0.4.8
+    image: langgenius/dify-api:0.4.9
     restart: always
     environment:
       # Startup mode, 'worker' starts the Celery worker for processing the queue.
@@ -202,7 +198,7 @@ services:
 
   # Frontend web application.
   web:
-    image: langgenius/dify-web:0.4.8
+    image: langgenius/dify-web:0.4.9
     restart: always
     environment:
       EDITION: SELF_HOSTED

web/Dockerfile

@@ -1,16 +1,19 @@
 # base image
-FROM node:18.17.0-alpine AS base
+FROM node:20.11.0-alpine AS base
+LABEL maintainer="takatost@gmail.com"
+
+RUN apk add --no-cache tzdata
+
 
 # install packages
 FROM base as packages
-LABEL maintainer="takatost@gmail.com"
 
 WORKDIR /app/web
 
 COPY package.json .
 COPY yarn.lock .
 
-RUN yarn --only=prod
+RUN yarn install --frozen-lockfile
 
 
 # build resources
@@ -32,6 +35,10 @@ ENV CONSOLE_API_URL http://127.0.0.1:5001
 ENV APP_API_URL http://127.0.0.1:5001
 ENV PORT 3000
 
+# set timezone
+ENV TZ UTC
+RUN ln -s /usr/share/zoneinfo/${TZ} /etc/localtime \
+    && echo ${TZ} > /etc/timezone
 
 WORKDIR /app/web
 COPY --from=builder /app/web/public ./public
@@ -40,10 +47,9 @@ COPY --from=builder /app/web/.next/static ./.next/static
 
 
 COPY docker/entrypoint.sh ./entrypoint.sh
-RUN chmod +x ./entrypoint.sh
 
 ARG COMMIT_SHA
 ENV COMMIT_SHA ${COMMIT_SHA}
 
 EXPOSE 3000
-ENTRYPOINT ["/bin/sh", "./entrypoint.sh"]
+ENTRYPOINT ["/bin/sh", "./entrypoint.sh"]

web/docker/entrypoint.sh

@@ -4,18 +4,8 @@ set -e
 
 export NEXT_PUBLIC_DEPLOY_ENV=${DEPLOY_ENV}
 export NEXT_PUBLIC_EDITION=${EDITION}
-
-if [[ -z "$CONSOLE_URL" ]]; then
-  export NEXT_PUBLIC_API_PREFIX=${CONSOLE_API_URL}/console/api
-else
-  export NEXT_PUBLIC_API_PREFIX=${CONSOLE_URL}/console/api
-fi
-
-if [[ -z "$APP_URL" ]]; then
-  export NEXT_PUBLIC_PUBLIC_API_PREFIX=${APP_API_URL}/api
-else
-  export NEXT_PUBLIC_PUBLIC_API_PREFIX=${APP_URL}/api
-fi
+export NEXT_PUBLIC_API_PREFIX=${CONSOLE_API_URL}/console/api
+export NEXT_PUBLIC_PUBLIC_API_PREFIX=${APP_API_URL}/api
 
 export NEXT_PUBLIC_SENTRY_DSN=${SENTRY_DSN}
 export NEXT_PUBLIC_SITE_ABOUT=${SITE_ABOUT}

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dify-web",
-  "version": "0.4.8",
+  "version": "0.4.9",
   "private": true,
   "scripts": {
     "dev": "next dev",