jandres/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2025-12-19 22:28:46 +00:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

move stopword data package from code to Dockerfile

Browse Source
This commit is contained in:
Byron Wang
2025-11-13 14:38:21 +08:00
parent 4e201ef059
commit e6f1500cfe
2 changed files with 3 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
api/Dockerfile
View File

@@ -73,7 +73,7 @@ COPY --from=packages ${VIRTUAL_ENV} ${VIRTUAL_ENV}
ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
# Download nltk data
RUN mkdir -p /usr/local/share/nltk_data && NLTK_DATA=/usr/local/share/nltk_data python -c "import nltk; nltk.download('punkt'); nltk.download('averaged_perceptron_tagger')" \
RUN mkdir -p /usr/local/share/nltk_data && NLTK_DATA=/usr/local/share/nltk_data python -c "import nltk; nltk.download('punkt'); nltk.download('averaged_perceptron_tagger'); nltk.download('stopwords')" \
&& chmod -R 755 /usr/local/share/nltk_data
ENV TIKTOKEN_CACHE_DIR=/app/api/.tiktoken_cache
@@ -95,7 +95,7 @@ RUN groupadd -r -g 1001 dify && \
ARG COMMIT_SHA
ENV COMMIT_SHA=${COMMIT_SHA}
ENV NLTK_DATA=/usr/local/share/nltk_data
USER 1001
ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]

3
api/core/rag/datasource/vdb/oracle/oraclevector.py
View File

@@ -302,8 +302,7 @@ class OracleVector(BaseVector):
nltk.data.find("tokenizers/punkt")
nltk.data.find("corpora/stopwords")
except LookupError:
nltk.download("punkt")
nltk.download("stopwords")
raise LookupError("Unable to find the required NLTK data package: punkt and stopwords")
e_str = re.sub(r"[^\w ]", "", query)
all_tokens = nltk.word_tokenize(e_str)
stop_words = stopwords.words("english")