Compare commits

..

3 Commits

Author SHA1 Message Date
yyh
c1e834742b Merge branch 'main' into 3-16-revert-changes 2026-03-16 19:57:42 +08:00
autofix-ci[bot]
9d2755eb85 [autofix.ci] apply automated fixes 2026-03-16 10:57:04 +00:00
Stephen Zhou
b80b0bd7a3 chore: bring back monaco-editor-react, revert #32966 2026-03-16 18:53:20 +08:00
45 changed files with 937 additions and 2127 deletions

View File

@@ -89,24 +89,14 @@ jobs:
- name: Merge reports
run: vp test --merge-reports --reporter=json --reporter=agent --coverage
- name: Report app/components baseline coverage
run: node ./scripts/report-components-coverage-baseline.mjs
- name: Report app/components test touch
env:
BASE_SHA: ${{ inputs.base_sha }}
DIFF_RANGE_MODE: ${{ inputs.diff_range_mode }}
HEAD_SHA: ${{ inputs.head_sha }}
run: node ./scripts/report-components-test-touch.mjs
- name: Check app/components pure diff coverage
- name: Check app/components diff coverage
env:
BASE_SHA: ${{ inputs.base_sha }}
DIFF_RANGE_MODE: ${{ inputs.diff_range_mode }}
HEAD_SHA: ${{ inputs.head_sha }}
run: node ./scripts/check-components-diff-coverage.mjs
- name: Check Coverage Summary
- name: Coverage Summary
if: always()
id: coverage-summary
run: |
@@ -115,15 +105,313 @@ jobs:
COVERAGE_FILE="coverage/coverage-final.json"
COVERAGE_SUMMARY_FILE="coverage/coverage-summary.json"
if [ -f "$COVERAGE_FILE" ] || [ -f "$COVERAGE_SUMMARY_FILE" ]; then
echo "has_coverage=true" >> "$GITHUB_OUTPUT"
if [ ! -f "$COVERAGE_FILE" ] && [ ! -f "$COVERAGE_SUMMARY_FILE" ]; then
echo "has_coverage=false" >> "$GITHUB_OUTPUT"
echo "### 🚨 Test Coverage Report :test_tube:" >> "$GITHUB_STEP_SUMMARY"
echo "Coverage data not found. Ensure Vitest runs with coverage enabled." >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
echo "has_coverage=false" >> "$GITHUB_OUTPUT"
echo "### 🚨 app/components Diff Coverage" >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "Coverage artifacts not found. Ensure Vitest merge reports ran with coverage enabled." >> "$GITHUB_STEP_SUMMARY"
echo "has_coverage=true" >> "$GITHUB_OUTPUT"
node <<'NODE' >> "$GITHUB_STEP_SUMMARY"
const fs = require('fs');
const path = require('path');
let libCoverage = null;
try {
libCoverage = require('istanbul-lib-coverage');
} catch (error) {
libCoverage = null;
}
const summaryPath = path.join('coverage', 'coverage-summary.json');
const finalPath = path.join('coverage', 'coverage-final.json');
const hasSummary = fs.existsSync(summaryPath);
const hasFinal = fs.existsSync(finalPath);
if (!hasSummary && !hasFinal) {
console.log('### Test Coverage Summary :test_tube:');
console.log('');
console.log('No coverage data found.');
process.exit(0);
}
const summary = hasSummary
? JSON.parse(fs.readFileSync(summaryPath, 'utf8'))
: null;
const coverage = hasFinal
? JSON.parse(fs.readFileSync(finalPath, 'utf8'))
: null;
const getLineCoverageFromStatements = (statementMap, statementHits) => {
const lineHits = {};
if (!statementMap || !statementHits) {
return lineHits;
}
Object.entries(statementMap).forEach(([key, statement]) => {
const line = statement?.start?.line;
if (!line) {
return;
}
const hits = statementHits[key] ?? 0;
const previous = lineHits[line];
lineHits[line] = previous === undefined ? hits : Math.max(previous, hits);
});
return lineHits;
};
const getFileCoverage = (entry) => (
libCoverage ? libCoverage.createFileCoverage(entry) : null
);
const getLineHits = (entry, fileCoverage) => {
const lineHits = entry.l ?? {};
if (Object.keys(lineHits).length > 0) {
return lineHits;
}
if (fileCoverage) {
return fileCoverage.getLineCoverage();
}
return getLineCoverageFromStatements(entry.statementMap ?? {}, entry.s ?? {});
};
const getUncoveredLines = (entry, fileCoverage, lineHits) => {
if (lineHits && Object.keys(lineHits).length > 0) {
return Object.entries(lineHits)
.filter(([, count]) => count === 0)
.map(([line]) => Number(line))
.sort((a, b) => a - b);
}
if (fileCoverage) {
return fileCoverage.getUncoveredLines();
}
return [];
};
const totals = {
lines: { covered: 0, total: 0 },
statements: { covered: 0, total: 0 },
branches: { covered: 0, total: 0 },
functions: { covered: 0, total: 0 },
};
const fileSummaries = [];
if (summary) {
const totalEntry = summary.total ?? {};
['lines', 'statements', 'branches', 'functions'].forEach((key) => {
if (totalEntry[key]) {
totals[key].covered = totalEntry[key].covered ?? 0;
totals[key].total = totalEntry[key].total ?? 0;
}
});
Object.entries(summary)
.filter(([file]) => file !== 'total')
.forEach(([file, data]) => {
fileSummaries.push({
file,
pct: data.lines?.pct ?? data.statements?.pct ?? 0,
lines: {
covered: data.lines?.covered ?? 0,
total: data.lines?.total ?? 0,
},
});
});
} else if (coverage) {
Object.entries(coverage).forEach(([file, entry]) => {
const fileCoverage = getFileCoverage(entry);
const lineHits = getLineHits(entry, fileCoverage);
const statementHits = entry.s ?? {};
const branchHits = entry.b ?? {};
const functionHits = entry.f ?? {};
const lineTotal = Object.keys(lineHits).length;
const lineCovered = Object.values(lineHits).filter((n) => n > 0).length;
const statementTotal = Object.keys(statementHits).length;
const statementCovered = Object.values(statementHits).filter((n) => n > 0).length;
const branchTotal = Object.values(branchHits).reduce((acc, branches) => acc + branches.length, 0);
const branchCovered = Object.values(branchHits).reduce(
(acc, branches) => acc + branches.filter((n) => n > 0).length,
0,
);
const functionTotal = Object.keys(functionHits).length;
const functionCovered = Object.values(functionHits).filter((n) => n > 0).length;
totals.lines.total += lineTotal;
totals.lines.covered += lineCovered;
totals.statements.total += statementTotal;
totals.statements.covered += statementCovered;
totals.branches.total += branchTotal;
totals.branches.covered += branchCovered;
totals.functions.total += functionTotal;
totals.functions.covered += functionCovered;
const pct = (covered, tot) => (tot > 0 ? (covered / tot) * 100 : 0);
fileSummaries.push({
file,
pct: pct(lineCovered || statementCovered, lineTotal || statementTotal),
lines: {
covered: lineCovered || statementCovered,
total: lineTotal || statementTotal,
},
});
});
}
const pct = (covered, tot) => (tot > 0 ? ((covered / tot) * 100).toFixed(2) : '0.00');
console.log('### Test Coverage Summary :test_tube:');
console.log('');
console.log('| Metric | Coverage | Covered / Total |');
console.log('|--------|----------|-----------------|');
console.log(`| Lines | ${pct(totals.lines.covered, totals.lines.total)}% | ${totals.lines.covered} / ${totals.lines.total} |`);
console.log(`| Statements | ${pct(totals.statements.covered, totals.statements.total)}% | ${totals.statements.covered} / ${totals.statements.total} |`);
console.log(`| Branches | ${pct(totals.branches.covered, totals.branches.total)}% | ${totals.branches.covered} / ${totals.branches.total} |`);
console.log(`| Functions | ${pct(totals.functions.covered, totals.functions.total)}% | ${totals.functions.covered} / ${totals.functions.total} |`);
console.log('');
console.log('<details><summary>File coverage (lowest lines first)</summary>');
console.log('');
console.log('```');
fileSummaries
.sort((a, b) => (a.pct - b.pct) || (b.lines.total - a.lines.total))
.slice(0, 25)
.forEach(({ file, pct, lines }) => {
console.log(`${pct.toFixed(2)}%\t${lines.covered}/${lines.total}\t${file}`);
});
console.log('```');
console.log('</details>');
if (coverage) {
const pctValue = (covered, tot) => {
if (tot === 0) {
return '0';
}
return ((covered / tot) * 100)
.toFixed(2)
.replace(/\.?0+$/, '');
};
const formatLineRanges = (lines) => {
if (lines.length === 0) {
return '';
}
const ranges = [];
let start = lines[0];
let end = lines[0];
for (let i = 1; i < lines.length; i += 1) {
const current = lines[i];
if (current === end + 1) {
end = current;
continue;
}
ranges.push(start === end ? `${start}` : `${start}-${end}`);
start = current;
end = current;
}
ranges.push(start === end ? `${start}` : `${start}-${end}`);
return ranges.join(',');
};
const tableTotals = {
statements: { covered: 0, total: 0 },
branches: { covered: 0, total: 0 },
functions: { covered: 0, total: 0 },
lines: { covered: 0, total: 0 },
};
const tableRows = Object.entries(coverage)
.map(([file, entry]) => {
const fileCoverage = getFileCoverage(entry);
const lineHits = getLineHits(entry, fileCoverage);
const statementHits = entry.s ?? {};
const branchHits = entry.b ?? {};
const functionHits = entry.f ?? {};
const lineTotal = Object.keys(lineHits).length;
const lineCovered = Object.values(lineHits).filter((n) => n > 0).length;
const statementTotal = Object.keys(statementHits).length;
const statementCovered = Object.values(statementHits).filter((n) => n > 0).length;
const branchTotal = Object.values(branchHits).reduce((acc, branches) => acc + branches.length, 0);
const branchCovered = Object.values(branchHits).reduce(
(acc, branches) => acc + branches.filter((n) => n > 0).length,
0,
);
const functionTotal = Object.keys(functionHits).length;
const functionCovered = Object.values(functionHits).filter((n) => n > 0).length;
tableTotals.lines.total += lineTotal;
tableTotals.lines.covered += lineCovered;
tableTotals.statements.total += statementTotal;
tableTotals.statements.covered += statementCovered;
tableTotals.branches.total += branchTotal;
tableTotals.branches.covered += branchCovered;
tableTotals.functions.total += functionTotal;
tableTotals.functions.covered += functionCovered;
const uncoveredLines = getUncoveredLines(entry, fileCoverage, lineHits);
const filePath = entry.path ?? file;
const relativePath = path.isAbsolute(filePath)
? path.relative(process.cwd(), filePath)
: filePath;
return {
file: relativePath || file,
statements: pctValue(statementCovered, statementTotal),
branches: pctValue(branchCovered, branchTotal),
functions: pctValue(functionCovered, functionTotal),
lines: pctValue(lineCovered, lineTotal),
uncovered: formatLineRanges(uncoveredLines),
};
})
.sort((a, b) => a.file.localeCompare(b.file));
const columns = [
{ key: 'file', header: 'File', align: 'left' },
{ key: 'statements', header: '% Stmts', align: 'right' },
{ key: 'branches', header: '% Branch', align: 'right' },
{ key: 'functions', header: '% Funcs', align: 'right' },
{ key: 'lines', header: '% Lines', align: 'right' },
{ key: 'uncovered', header: 'Uncovered Line #s', align: 'left' },
];
const allFilesRow = {
file: 'All files',
statements: pctValue(tableTotals.statements.covered, tableTotals.statements.total),
branches: pctValue(tableTotals.branches.covered, tableTotals.branches.total),
functions: pctValue(tableTotals.functions.covered, tableTotals.functions.total),
lines: pctValue(tableTotals.lines.covered, tableTotals.lines.total),
uncovered: '',
};
const rowsForOutput = [allFilesRow, ...tableRows];
const formatRow = (row) => `| ${columns
.map(({ key }) => String(row[key] ?? ''))
.join(' | ')} |`;
const headerRow = `| ${columns.map(({ header }) => header).join(' | ')} |`;
const dividerRow = `| ${columns
.map(({ align }) => (align === 'right' ? '---:' : ':---'))
.join(' | ')} |`;
console.log('');
console.log('<details><summary>Vitest coverage table</summary>');
console.log('');
console.log(headerRow);
console.log(dividerRow);
rowsForOutput.forEach((row) => console.log(formatRow(row)));
console.log('</details>');
}
NODE
- name: Upload Coverage Artifact
if: steps.coverage-summary.outputs.has_coverage == 'true'

View File

@@ -88,8 +88,6 @@ def clean_workflow_runs(
"""
Clean workflow runs and related workflow data for free tenants.
"""
from extensions.otel.runtime import flush_telemetry
if (start_from is None) ^ (end_before is None):
raise click.UsageError("--start-from and --end-before must be provided together.")
@@ -106,27 +104,16 @@ def clean_workflow_runs(
end_before = now - datetime.timedelta(days=to_days_ago)
before_days = 0
if from_days_ago is not None and to_days_ago is not None:
task_label = f"{from_days_ago}to{to_days_ago}"
elif start_from is None:
task_label = f"before-{before_days}"
else:
task_label = "custom"
start_time = datetime.datetime.now(datetime.UTC)
click.echo(click.style(f"Starting workflow run cleanup at {start_time.isoformat()}.", fg="white"))
try:
WorkflowRunCleanup(
days=before_days,
batch_size=batch_size,
start_from=start_from,
end_before=end_before,
dry_run=dry_run,
task_label=task_label,
).run()
finally:
flush_telemetry()
WorkflowRunCleanup(
days=before_days,
batch_size=batch_size,
start_from=start_from,
end_before=end_before,
dry_run=dry_run,
).run()
end_time = datetime.datetime.now(datetime.UTC)
elapsed = end_time - start_time
@@ -672,8 +659,6 @@ def clean_expired_messages(
"""
Clean expired messages and related data for tenants based on clean policy.
"""
from extensions.otel.runtime import flush_telemetry
click.echo(click.style("clean_messages: start clean messages.", fg="green"))
start_at = time.perf_counter()
@@ -713,13 +698,6 @@ def clean_expired_messages(
# NOTE: graceful_period will be ignored when billing is disabled.
policy = create_message_clean_policy(graceful_period_days=graceful_period)
if from_days_ago is not None and before_days is not None:
task_label = f"{from_days_ago}to{before_days}"
elif start_from is None and before_days is not None:
task_label = f"before-{before_days}"
else:
task_label = "custom"
# Create and run the cleanup service
if abs_mode:
assert start_from is not None
@@ -730,7 +708,6 @@ def clean_expired_messages(
end_before=end_before,
batch_size=batch_size,
dry_run=dry_run,
task_label=task_label,
)
elif from_days_ago is None:
assert before_days is not None
@@ -739,7 +716,6 @@ def clean_expired_messages(
days=before_days,
batch_size=batch_size,
dry_run=dry_run,
task_label=task_label,
)
else:
assert before_days is not None
@@ -751,7 +727,6 @@ def clean_expired_messages(
end_before=now - datetime.timedelta(days=before_days),
batch_size=batch_size,
dry_run=dry_run,
task_label=task_label,
)
stats = service.run()
@@ -777,8 +752,6 @@ def clean_expired_messages(
)
)
raise
finally:
flush_telemetry()
click.echo(click.style("messages cleanup completed.", fg="green"))

View File

@@ -5,7 +5,6 @@ import re
import threading
import time
import uuid
from collections.abc import Mapping
from typing import Any
from flask import Flask, current_app
@@ -38,7 +37,7 @@ from extensions.ext_storage import storage
from libs import helper
from libs.datetime_utils import naive_utc_now
from models import Account
from models.dataset import AutomaticRulesConfig, ChildChunk, Dataset, DatasetProcessRule, DocumentSegment
from models.dataset import ChildChunk, Dataset, DatasetProcessRule, DocumentSegment
from models.dataset import Document as DatasetDocument
from models.model import UploadFile
from services.feature_service import FeatureService
@@ -266,7 +265,7 @@ class IndexingRunner:
self,
tenant_id: str,
extract_settings: list[ExtractSetting],
tmp_processing_rule: Mapping[str, Any],
tmp_processing_rule: dict,
doc_form: str | None = None,
doc_language: str = "English",
dataset_id: str | None = None,
@@ -377,7 +376,7 @@ class IndexingRunner:
return IndexingEstimate(total_segments=total_segments, preview=preview_texts)
def _extract(
self, index_processor: BaseIndexProcessor, dataset_document: DatasetDocument, process_rule: Mapping[str, Any]
self, index_processor: BaseIndexProcessor, dataset_document: DatasetDocument, process_rule: dict
) -> list[Document]:
data_source_info = dataset_document.data_source_info_dict
text_docs = []
@@ -544,7 +543,6 @@ class IndexingRunner:
"""
Clean the document text according to the processing rules.
"""
rules: AutomaticRulesConfig | dict[str, Any]
if processing_rule.mode == "automatic":
rules = DatasetProcessRule.AUTOMATIC_RULES
else:
@@ -758,7 +756,7 @@ class IndexingRunner:
dataset: Dataset,
text_docs: list[Document],
doc_language: str,
process_rule: Mapping[str, Any],
process_rule: dict,
current_user: Account | None = None,
) -> list[Document]:
# get embedding model instance

View File

@@ -6,5 +6,6 @@ of responses based on upstream node outputs and constants.
"""
from .coordinator import ResponseStreamCoordinator
from .session import RESPONSE_SESSION_NODE_TYPES
__all__ = ["ResponseStreamCoordinator"]
__all__ = ["RESPONSE_SESSION_NODE_TYPES", "ResponseStreamCoordinator"]

View File

@@ -3,6 +3,10 @@ Internal response session management for response coordinator.
This module contains the private ResponseSession class used internally
by ResponseStreamCoordinator to manage streaming sessions.
`RESPONSE_SESSION_NODE_TYPES` is intentionally mutable so downstream applications
can opt additional response-capable node types into session creation without
patching the coordinator.
"""
from __future__ import annotations
@@ -10,6 +14,7 @@ from __future__ import annotations
from dataclasses import dataclass
from typing import Protocol, cast
from dify_graph.enums import BuiltinNodeTypes, NodeType
from dify_graph.nodes.base.template import Template
from dify_graph.runtime.graph_runtime_state import NodeProtocol
@@ -20,6 +25,12 @@ class _ResponseSessionNodeProtocol(NodeProtocol, Protocol):
def get_streaming_template(self) -> Template: ...
RESPONSE_SESSION_NODE_TYPES: list[NodeType] = [
BuiltinNodeTypes.ANSWER,
BuiltinNodeTypes.END,
]
@dataclass
class ResponseSession:
"""
@@ -38,8 +49,8 @@ class ResponseSession:
Create a ResponseSession from a response-capable node.
The parameter is typed as `NodeProtocol` because the graph is exposed behind a protocol at the runtime layer.
At runtime this must be a node that implements `get_streaming_template()`. The coordinator decides which
graph nodes should be treated as response-capable before they reach this factory.
At runtime this must be a node whose `node_type` is listed in `RESPONSE_SESSION_NODE_TYPES`
and which implements `get_streaming_template()`.
Args:
node: Node from the materialized workflow graph.
@@ -48,8 +59,15 @@ class ResponseSession:
ResponseSession configured with the node's streaming template
Raises:
TypeError: If node does not implement the response-session streaming contract.
TypeError: If node is not a supported response node type.
"""
if node.node_type not in RESPONSE_SESSION_NODE_TYPES:
supported_node_types = ", ".join(RESPONSE_SESSION_NODE_TYPES)
raise TypeError(
"ResponseSession.from_node only supports node types in "
f"RESPONSE_SESSION_NODE_TYPES: {supported_node_types}"
)
response_node = cast(_ResponseSessionNodeProtocol, node)
try:
template = response_node.get_streaming_template()

View File

@@ -5,7 +5,7 @@ from typing import Union
from celery.signals import worker_init
from flask_login import user_loaded_from_request, user_logged_in
from opentelemetry import metrics, trace
from opentelemetry import trace
from opentelemetry.propagate import set_global_textmap
from opentelemetry.propagators.b3 import B3MultiFormat
from opentelemetry.propagators.composite import CompositePropagator
@@ -31,29 +31,9 @@ def setup_context_propagation() -> None:
def shutdown_tracer() -> None:
flush_telemetry()
def flush_telemetry() -> None:
"""
Best-effort flush for telemetry providers.
This is mainly used by short-lived command processes (e.g. Kubernetes CronJob)
so counters/histograms are exported before the process exits.
"""
provider = trace.get_tracer_provider()
if hasattr(provider, "force_flush"):
try:
provider.force_flush()
except Exception:
logger.exception("otel: failed to flush trace provider")
metric_provider = metrics.get_meter_provider()
if hasattr(metric_provider, "force_flush"):
try:
metric_provider.force_flush()
except Exception:
logger.exception("otel: failed to flush metric provider")
provider.force_flush()
def is_celery_worker():

View File

@@ -10,7 +10,7 @@ import re
import time
from datetime import datetime
from json import JSONDecodeError
from typing import Any, TypedDict, cast
from typing import Any, cast
from uuid import uuid4
import sqlalchemy as sa
@@ -37,61 +37,6 @@ from .types import AdjustedJSON, BinaryData, EnumText, LongText, StringUUID, adj
logger = logging.getLogger(__name__)
class PreProcessingRuleItem(TypedDict):
id: str
enabled: bool
class SegmentationConfig(TypedDict):
delimiter: str
max_tokens: int
chunk_overlap: int
class AutomaticRulesConfig(TypedDict):
pre_processing_rules: list[PreProcessingRuleItem]
segmentation: SegmentationConfig
class ProcessRuleDict(TypedDict):
id: str
dataset_id: str
mode: str
rules: dict[str, Any] | None
class DocMetadataDetailItem(TypedDict):
id: str
name: str
type: str
value: Any
class AttachmentItem(TypedDict):
id: str
name: str
size: int
extension: str
mime_type: str
source_url: str
class DatasetBindingItem(TypedDict):
id: str
name: str
class ExternalKnowledgeApiDict(TypedDict):
id: str
tenant_id: str
name: str
description: str
settings: dict[str, Any] | None
dataset_bindings: list[DatasetBindingItem]
created_by: str
created_at: str
class DatasetPermissionEnum(enum.StrEnum):
ONLY_ME = "only_me"
ALL_TEAM = "all_team_members"
@@ -389,7 +334,7 @@ class DatasetProcessRule(Base): # bug
MODES = ["automatic", "custom", "hierarchical"]
PRE_PROCESSING_RULES = ["remove_stopwords", "remove_extra_spaces", "remove_urls_emails"]
AUTOMATIC_RULES: AutomaticRulesConfig = {
AUTOMATIC_RULES: dict[str, Any] = {
"pre_processing_rules": [
{"id": "remove_extra_spaces", "enabled": True},
{"id": "remove_urls_emails", "enabled": False},
@@ -397,7 +342,7 @@ class DatasetProcessRule(Base): # bug
"segmentation": {"delimiter": "\n", "max_tokens": 500, "chunk_overlap": 50},
}
def to_dict(self) -> ProcessRuleDict:
def to_dict(self) -> dict[str, Any]:
return {
"id": self.id,
"dataset_id": self.dataset_id,
@@ -586,7 +531,7 @@ class Document(Base):
return self.updated_at
@property
def doc_metadata_details(self) -> list[DocMetadataDetailItem] | None:
def doc_metadata_details(self) -> list[dict[str, Any]] | None:
if self.doc_metadata:
document_metadatas = (
db.session.query(DatasetMetadata)
@@ -596,9 +541,9 @@ class Document(Base):
)
.all()
)
metadata_list: list[DocMetadataDetailItem] = []
metadata_list: list[dict[str, Any]] = []
for metadata in document_metadatas:
metadata_dict: DocMetadataDetailItem = {
metadata_dict: dict[str, Any] = {
"id": metadata.id,
"name": metadata.name,
"type": metadata.type,
@@ -612,13 +557,13 @@ class Document(Base):
return None
@property
def process_rule_dict(self) -> ProcessRuleDict | None:
def process_rule_dict(self) -> dict[str, Any] | None:
if self.dataset_process_rule_id and self.dataset_process_rule:
return self.dataset_process_rule.to_dict()
return None
def get_built_in_fields(self) -> list[DocMetadataDetailItem]:
built_in_fields: list[DocMetadataDetailItem] = []
def get_built_in_fields(self) -> list[dict[str, Any]]:
built_in_fields: list[dict[str, Any]] = []
built_in_fields.append(
{
"id": "built-in",
@@ -932,7 +877,7 @@ class DocumentSegment(Base):
return text
@property
def attachments(self) -> list[AttachmentItem]:
def attachments(self) -> list[dict[str, Any]]:
# Use JOIN to fetch attachments in a single query instead of two separate queries
attachments_with_bindings = db.session.execute(
select(SegmentAttachmentBinding, UploadFile)
@@ -946,7 +891,7 @@ class DocumentSegment(Base):
).all()
if not attachments_with_bindings:
return []
attachment_list: list[AttachmentItem] = []
attachment_list = []
for _, attachment in attachments_with_bindings:
upload_file_id = attachment.id
nonce = os.urandom(16).hex()
@@ -1316,7 +1261,7 @@ class ExternalKnowledgeApis(TypeBase):
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
)
def to_dict(self) -> ExternalKnowledgeApiDict:
def to_dict(self) -> dict[str, Any]:
return {
"id": self.id,
"tenant_id": self.tenant_id,
@@ -1336,13 +1281,13 @@ class ExternalKnowledgeApis(TypeBase):
return None
@property
def dataset_bindings(self) -> list[DatasetBindingItem]:
def dataset_bindings(self) -> list[dict[str, Any]]:
external_knowledge_bindings = db.session.scalars(
select(ExternalKnowledgeBindings).where(ExternalKnowledgeBindings.external_knowledge_api_id == self.id)
).all()
dataset_ids = [binding.dataset_id for binding in external_knowledge_bindings]
datasets = db.session.scalars(select(Dataset).where(Dataset.id.in_(dataset_ids))).all()
dataset_bindings: list[DatasetBindingItem] = []
dataset_bindings: list[dict[str, Any]] = []
for dataset in datasets:
dataset_bindings.append({"id": dataset.id, "name": dataset.name})

View File

@@ -3,7 +3,7 @@ import time
from collections.abc import Mapping
from datetime import datetime
from functools import cached_property
from typing import Any, TypedDict, cast
from typing import Any, cast
from uuid import uuid4
import sqlalchemy as sa
@@ -24,44 +24,6 @@ from .model import Account
from .types import EnumText, LongText, StringUUID
class WorkflowTriggerLogDict(TypedDict):
id: str
tenant_id: str
app_id: str
workflow_id: str
workflow_run_id: str | None
root_node_id: str | None
trigger_metadata: Any
trigger_type: str
trigger_data: Any
inputs: Any
outputs: Any
status: str
error: str | None
queue_name: str
celery_task_id: str | None
retry_count: int
elapsed_time: float | None
total_tokens: int | None
created_by_role: str
created_by: str
created_at: str | None
triggered_at: str | None
finished_at: str | None
class WorkflowSchedulePlanDict(TypedDict):
id: str
app_id: str
node_id: str
tenant_id: str
cron_expression: str
timezone: str
next_run_at: str | None
created_at: str
updated_at: str
class TriggerSubscription(TypeBase):
"""
Trigger provider model for managing credentials
@@ -288,7 +250,7 @@ class WorkflowTriggerLog(TypeBase):
created_by_role = CreatorUserRole(self.created_by_role)
return db.session.get(EndUser, self.created_by) if created_by_role == CreatorUserRole.END_USER else None
def to_dict(self) -> WorkflowTriggerLogDict:
def to_dict(self) -> dict[str, Any]:
"""Convert to dictionary for API responses"""
return {
"id": self.id,
@@ -519,7 +481,7 @@ class WorkflowSchedulePlan(TypeBase):
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
)
def to_dict(self) -> WorkflowSchedulePlanDict:
def to_dict(self) -> dict[str, Any]:
"""Convert to dictionary representation"""
return {
"id": self.id,

View File

@@ -3,7 +3,7 @@ import logging
from collections.abc import Generator, Mapping, Sequence
from datetime import datetime
from enum import StrEnum
from typing import TYPE_CHECKING, Any, Optional, TypedDict, Union, cast
from typing import TYPE_CHECKING, Any, Optional, Union, cast
from uuid import uuid4
import sqlalchemy as sa
@@ -60,22 +60,6 @@ from .types import EnumText, LongText, StringUUID
logger = logging.getLogger(__name__)
class WorkflowContentDict(TypedDict):
graph: Mapping[str, Any]
features: dict[str, Any]
environment_variables: list[dict[str, Any]]
conversation_variables: list[dict[str, Any]]
rag_pipeline_variables: list[dict[str, Any]]
class WorkflowRunSummaryDict(TypedDict):
id: str
status: str
triggered_from: str
elapsed_time: float
total_tokens: int
class WorkflowType(StrEnum):
"""
Workflow Type Enum
@@ -518,14 +502,14 @@ class Workflow(Base): # bug
)
self._environment_variables = environment_variables_json
def to_dict(self, *, include_secret: bool = False) -> WorkflowContentDict:
def to_dict(self, *, include_secret: bool = False) -> Mapping[str, Any]:
environment_variables = list(self.environment_variables)
environment_variables = [
v if not isinstance(v, SecretVariable) or include_secret else v.model_copy(update={"value": ""})
for v in environment_variables
]
result: WorkflowContentDict = {
result = {
"graph": self.graph_dict,
"features": self.features_dict,
"environment_variables": [var.model_dump(mode="json") for var in environment_variables],
@@ -1247,7 +1231,7 @@ class WorkflowArchiveLog(TypeBase):
)
@property
def workflow_run_summary(self) -> WorkflowRunSummaryDict:
def workflow_run_summary(self) -> dict[str, Any]:
return {
"id": self.workflow_run_id,
"status": self.run_status,

View File

@@ -18,7 +18,7 @@ from extensions.ext_database import db
from models.account import Account
from models.enums import CreatorUserRole, WorkflowTriggerStatus
from models.model import App, EndUser
from models.trigger import WorkflowTriggerLog, WorkflowTriggerLogDict
from models.trigger import WorkflowTriggerLog
from models.workflow import Workflow
from repositories.sqlalchemy_workflow_trigger_log_repository import SQLAlchemyWorkflowTriggerLogRepository
from services.errors.app import QuotaExceededError, WorkflowNotFoundError, WorkflowQuotaLimitError
@@ -224,9 +224,7 @@ class AsyncWorkflowService:
return cls.trigger_workflow_async(session, user, trigger_data)
@classmethod
def get_trigger_log(
cls, workflow_trigger_log_id: str, tenant_id: str | None = None
) -> WorkflowTriggerLogDict | None:
def get_trigger_log(cls, workflow_trigger_log_id: str, tenant_id: str | None = None) -> dict[str, Any] | None:
"""
Get trigger log by ID
@@ -249,7 +247,7 @@ class AsyncWorkflowService:
@classmethod
def get_recent_logs(
cls, tenant_id: str, app_id: str, hours: int = 24, limit: int = 100, offset: int = 0
) -> list[WorkflowTriggerLogDict]:
) -> list[dict[str, Any]]:
"""
Get recent trigger logs
@@ -274,7 +272,7 @@ class AsyncWorkflowService:
@classmethod
def get_failed_logs_for_retry(
cls, tenant_id: str, max_retry_count: int = 3, limit: int = 100
) -> list[WorkflowTriggerLogDict]:
) -> list[dict[str, Any]]:
"""
Get failed logs eligible for retry

View File

@@ -1,16 +1,16 @@
import datetime
import logging
import os
import random
import time
from collections.abc import Sequence
from typing import TYPE_CHECKING, cast
from typing import cast
import sqlalchemy as sa
from sqlalchemy import delete, select, tuple_
from sqlalchemy.engine import CursorResult
from sqlalchemy.orm import Session
from configs import dify_config
from extensions.ext_database import db
from libs.datetime_utils import naive_utc_now
from models.model import (
@@ -33,131 +33,6 @@ from services.retention.conversation.messages_clean_policy import (
logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from opentelemetry.metrics import Counter, Histogram
class MessagesCleanupMetrics:
"""
Records low-cardinality OpenTelemetry metrics for expired message cleanup jobs.
We keep labels stable (dry_run/window_mode/task_label/status) so these metrics remain
dashboard-friendly for long-running CronJob executions.
"""
_job_runs_total: "Counter | None"
_batches_total: "Counter | None"
_messages_scanned_total: "Counter | None"
_messages_filtered_total: "Counter | None"
_messages_deleted_total: "Counter | None"
_job_duration_seconds: "Histogram | None"
_batch_duration_seconds: "Histogram | None"
_base_attributes: dict[str, str]
def __init__(self, *, dry_run: bool, has_window: bool, task_label: str) -> None:
self._job_runs_total = None
self._batches_total = None
self._messages_scanned_total = None
self._messages_filtered_total = None
self._messages_deleted_total = None
self._job_duration_seconds = None
self._batch_duration_seconds = None
self._base_attributes = {
"job_name": "messages_cleanup",
"dry_run": str(dry_run).lower(),
"window_mode": "between" if has_window else "before_cutoff",
"task_label": task_label,
}
self._init_instruments()
def _init_instruments(self) -> None:
if not dify_config.ENABLE_OTEL:
return
try:
from opentelemetry.metrics import get_meter
meter = get_meter("messages_cleanup", version=dify_config.project.version)
self._job_runs_total = meter.create_counter(
"messages_cleanup_jobs_total",
description="Total number of expired message cleanup jobs by status.",
unit="{job}",
)
self._batches_total = meter.create_counter(
"messages_cleanup_batches_total",
description="Total number of message cleanup batches processed.",
unit="{batch}",
)
self._messages_scanned_total = meter.create_counter(
"messages_cleanup_scanned_messages_total",
description="Total messages scanned by cleanup jobs.",
unit="{message}",
)
self._messages_filtered_total = meter.create_counter(
"messages_cleanup_filtered_messages_total",
description="Total messages selected by cleanup policy.",
unit="{message}",
)
self._messages_deleted_total = meter.create_counter(
"messages_cleanup_deleted_messages_total",
description="Total messages deleted by cleanup jobs.",
unit="{message}",
)
self._job_duration_seconds = meter.create_histogram(
"messages_cleanup_job_duration_seconds",
description="Duration of expired message cleanup jobs in seconds.",
unit="s",
)
self._batch_duration_seconds = meter.create_histogram(
"messages_cleanup_batch_duration_seconds",
description="Duration of expired message cleanup batch processing in seconds.",
unit="s",
)
except Exception:
logger.exception("messages_cleanup_metrics: failed to initialize instruments")
def _attrs(self, **extra: str) -> dict[str, str]:
return {**self._base_attributes, **extra}
@staticmethod
def _add(counter: "Counter | None", value: int, attributes: dict[str, str]) -> None:
if not counter or value <= 0:
return
try:
counter.add(value, attributes)
except Exception:
logger.exception("messages_cleanup_metrics: failed to add counter value")
@staticmethod
def _record(histogram: "Histogram | None", value: float, attributes: dict[str, str]) -> None:
if not histogram:
return
try:
histogram.record(value, attributes)
except Exception:
logger.exception("messages_cleanup_metrics: failed to record histogram value")
def record_batch(
self,
*,
scanned_messages: int,
filtered_messages: int,
deleted_messages: int,
batch_duration_seconds: float,
) -> None:
attributes = self._attrs()
self._add(self._batches_total, 1, attributes)
self._add(self._messages_scanned_total, scanned_messages, attributes)
self._add(self._messages_filtered_total, filtered_messages, attributes)
self._add(self._messages_deleted_total, deleted_messages, attributes)
self._record(self._batch_duration_seconds, batch_duration_seconds, attributes)
def record_completion(self, *, status: str, job_duration_seconds: float) -> None:
attributes = self._attrs(status=status)
self._add(self._job_runs_total, 1, attributes)
self._record(self._job_duration_seconds, job_duration_seconds, attributes)
class MessagesCleanService:
"""
Service for cleaning expired messages based on retention policies.
@@ -173,7 +48,6 @@ class MessagesCleanService:
start_from: datetime.datetime | None = None,
batch_size: int = 1000,
dry_run: bool = False,
task_label: str = "custom",
) -> None:
"""
Initialize the service with cleanup parameters.
@@ -184,18 +58,12 @@ class MessagesCleanService:
start_from: Optional start time (inclusive) of the range
batch_size: Number of messages to process per batch
dry_run: Whether to perform a dry run (no actual deletion)
task_label: Optional task label for retention metrics
"""
self._policy = policy
self._end_before = end_before
self._start_from = start_from
self._batch_size = batch_size
self._dry_run = dry_run
self._metrics = MessagesCleanupMetrics(
dry_run=dry_run,
has_window=bool(start_from),
task_label=task_label,
)
@classmethod
def from_time_range(
@@ -205,7 +73,6 @@ class MessagesCleanService:
end_before: datetime.datetime,
batch_size: int = 1000,
dry_run: bool = False,
task_label: str = "custom",
) -> "MessagesCleanService":
"""
Create a service instance for cleaning messages within a specific time range.
@@ -218,7 +85,6 @@ class MessagesCleanService:
end_before: End time (exclusive) of the range
batch_size: Number of messages to process per batch
dry_run: Whether to perform a dry run (no actual deletion)
task_label: Optional task label for retention metrics
Returns:
MessagesCleanService instance
@@ -246,7 +112,6 @@ class MessagesCleanService:
start_from=start_from,
batch_size=batch_size,
dry_run=dry_run,
task_label=task_label,
)
@classmethod
@@ -256,7 +121,6 @@ class MessagesCleanService:
days: int = 30,
batch_size: int = 1000,
dry_run: bool = False,
task_label: str = "custom",
) -> "MessagesCleanService":
"""
Create a service instance for cleaning messages older than specified days.
@@ -266,7 +130,6 @@ class MessagesCleanService:
days: Number of days to look back from now
batch_size: Number of messages to process per batch
dry_run: Whether to perform a dry run (no actual deletion)
task_label: Optional task label for retention metrics
Returns:
MessagesCleanService instance
@@ -290,14 +153,7 @@ class MessagesCleanService:
policy.__class__.__name__,
)
return cls(
policy=policy,
end_before=end_before,
start_from=None,
batch_size=batch_size,
dry_run=dry_run,
task_label=task_label,
)
return cls(policy=policy, end_before=end_before, start_from=None, batch_size=batch_size, dry_run=dry_run)
def run(self) -> dict[str, int]:
"""
@@ -306,18 +162,7 @@ class MessagesCleanService:
Returns:
Dict with statistics: batches, filtered_messages, total_deleted
"""
status = "success"
run_start = time.monotonic()
try:
return self._clean_messages_by_time_range()
except Exception:
status = "failed"
raise
finally:
self._metrics.record_completion(
status=status,
job_duration_seconds=time.monotonic() - run_start,
)
return self._clean_messages_by_time_range()
def _clean_messages_by_time_range(self) -> dict[str, int]:
"""
@@ -352,14 +197,11 @@ class MessagesCleanService:
self._end_before,
)
max_batch_interval_ms = dify_config.SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_MAX_INTERVAL
max_batch_interval_ms = int(os.environ.get("SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_MAX_INTERVAL", 200))
while True:
stats["batches"] += 1
batch_start = time.monotonic()
batch_scanned_messages = 0
batch_filtered_messages = 0
batch_deleted_messages = 0
# Step 1: Fetch a batch of messages using cursor
with Session(db.engine, expire_on_commit=False) as session:
@@ -398,16 +240,9 @@ class MessagesCleanService:
# Track total messages fetched across all batches
stats["total_messages"] += len(messages)
batch_scanned_messages = len(messages)
if not messages:
logger.info("clean_messages (batch %s): no more messages to process", stats["batches"])
self._metrics.record_batch(
scanned_messages=batch_scanned_messages,
filtered_messages=batch_filtered_messages,
deleted_messages=batch_deleted_messages,
batch_duration_seconds=time.monotonic() - batch_start,
)
break
# Update cursor to the last message's (created_at, id)
@@ -433,12 +268,6 @@ class MessagesCleanService:
if not apps:
logger.info("clean_messages (batch %s): no apps found, skip", stats["batches"])
self._metrics.record_batch(
scanned_messages=batch_scanned_messages,
filtered_messages=batch_filtered_messages,
deleted_messages=batch_deleted_messages,
batch_duration_seconds=time.monotonic() - batch_start,
)
continue
# Build app_id -> tenant_id mapping
@@ -457,16 +286,9 @@ class MessagesCleanService:
if not message_ids_to_delete:
logger.info("clean_messages (batch %s): no messages to delete, skip", stats["batches"])
self._metrics.record_batch(
scanned_messages=batch_scanned_messages,
filtered_messages=batch_filtered_messages,
deleted_messages=batch_deleted_messages,
batch_duration_seconds=time.monotonic() - batch_start,
)
continue
stats["filtered_messages"] += len(message_ids_to_delete)
batch_filtered_messages = len(message_ids_to_delete)
# Step 4: Batch delete messages and their relations
if not self._dry_run:
@@ -487,7 +309,6 @@ class MessagesCleanService:
commit_ms = int((time.monotonic() - commit_start) * 1000)
stats["total_deleted"] += messages_deleted
batch_deleted_messages = messages_deleted
logger.info(
"clean_messages (batch %s): processed %s messages, deleted %s messages",
@@ -522,13 +343,6 @@ class MessagesCleanService:
for msg_id in sampled_ids:
logger.info("clean_messages (batch %s, dry_run) sample: message_id=%s", stats["batches"], msg_id)
self._metrics.record_batch(
scanned_messages=batch_scanned_messages,
filtered_messages=batch_filtered_messages,
deleted_messages=batch_deleted_messages,
batch_duration_seconds=time.monotonic() - batch_start,
)
logger.info(
"clean_messages completed: total batches: %s, total messages: %s, filtered messages: %s, total deleted: %s",
stats["batches"],

View File

@@ -1,9 +1,9 @@
import datetime
import logging
import os
import random
import time
from collections.abc import Iterable, Sequence
from typing import TYPE_CHECKING
import click
from sqlalchemy.orm import Session, sessionmaker
@@ -20,159 +20,6 @@ from services.billing_service import BillingService, SubscriptionPlan
logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from opentelemetry.metrics import Counter, Histogram
class WorkflowRunCleanupMetrics:
"""
Records low-cardinality OpenTelemetry metrics for workflow run cleanup jobs.
Metrics are emitted with stable labels only (dry_run/window_mode/task_label/status)
to keep dashboard and alert cardinality predictable in production clusters.
"""
_job_runs_total: "Counter | None"
_batches_total: "Counter | None"
_runs_scanned_total: "Counter | None"
_runs_targeted_total: "Counter | None"
_runs_deleted_total: "Counter | None"
_runs_skipped_total: "Counter | None"
_related_records_total: "Counter | None"
_job_duration_seconds: "Histogram | None"
_batch_duration_seconds: "Histogram | None"
_base_attributes: dict[str, str]
def __init__(self, *, dry_run: bool, has_window: bool, task_label: str) -> None:
self._job_runs_total = None
self._batches_total = None
self._runs_scanned_total = None
self._runs_targeted_total = None
self._runs_deleted_total = None
self._runs_skipped_total = None
self._related_records_total = None
self._job_duration_seconds = None
self._batch_duration_seconds = None
self._base_attributes = {
"job_name": "workflow_run_cleanup",
"dry_run": str(dry_run).lower(),
"window_mode": "between" if has_window else "before_cutoff",
"task_label": task_label,
}
self._init_instruments()
def _init_instruments(self) -> None:
if not dify_config.ENABLE_OTEL:
return
try:
from opentelemetry.metrics import get_meter
meter = get_meter("workflow_run_cleanup", version=dify_config.project.version)
self._job_runs_total = meter.create_counter(
"workflow_run_cleanup_jobs_total",
description="Total number of workflow run cleanup jobs by status.",
unit="{job}",
)
self._batches_total = meter.create_counter(
"workflow_run_cleanup_batches_total",
description="Total number of processed cleanup batches.",
unit="{batch}",
)
self._runs_scanned_total = meter.create_counter(
"workflow_run_cleanup_scanned_runs_total",
description="Total workflow runs scanned by cleanup jobs.",
unit="{run}",
)
self._runs_targeted_total = meter.create_counter(
"workflow_run_cleanup_targeted_runs_total",
description="Total workflow runs targeted by cleanup policy.",
unit="{run}",
)
self._runs_deleted_total = meter.create_counter(
"workflow_run_cleanup_deleted_runs_total",
description="Total workflow runs deleted by cleanup jobs.",
unit="{run}",
)
self._runs_skipped_total = meter.create_counter(
"workflow_run_cleanup_skipped_runs_total",
description="Total workflow runs skipped because tenant is paid/unknown.",
unit="{run}",
)
self._related_records_total = meter.create_counter(
"workflow_run_cleanup_related_records_total",
description="Total related records processed by cleanup jobs.",
unit="{record}",
)
self._job_duration_seconds = meter.create_histogram(
"workflow_run_cleanup_job_duration_seconds",
description="Duration of workflow run cleanup jobs in seconds.",
unit="s",
)
self._batch_duration_seconds = meter.create_histogram(
"workflow_run_cleanup_batch_duration_seconds",
description="Duration of workflow run cleanup batch processing in seconds.",
unit="s",
)
except Exception:
logger.exception("workflow_run_cleanup_metrics: failed to initialize instruments")
def _attrs(self, **extra: str) -> dict[str, str]:
return {**self._base_attributes, **extra}
@staticmethod
def _add(counter: "Counter | None", value: int, attributes: dict[str, str]) -> None:
if not counter or value <= 0:
return
try:
counter.add(value, attributes)
except Exception:
logger.exception("workflow_run_cleanup_metrics: failed to add counter value")
@staticmethod
def _record(histogram: "Histogram | None", value: float, attributes: dict[str, str]) -> None:
if not histogram:
return
try:
histogram.record(value, attributes)
except Exception:
logger.exception("workflow_run_cleanup_metrics: failed to record histogram value")
def record_batch(
self,
*,
batch_rows: int,
targeted_runs: int,
skipped_runs: int,
deleted_runs: int,
related_counts: dict[str, int] | None,
related_action: str | None,
batch_duration_seconds: float,
) -> None:
attributes = self._attrs()
self._add(self._batches_total, 1, attributes)
self._add(self._runs_scanned_total, batch_rows, attributes)
self._add(self._runs_targeted_total, targeted_runs, attributes)
self._add(self._runs_skipped_total, skipped_runs, attributes)
self._add(self._runs_deleted_total, deleted_runs, attributes)
self._record(self._batch_duration_seconds, batch_duration_seconds, attributes)
if not related_counts or not related_action:
return
for record_type, count in related_counts.items():
self._add(
self._related_records_total,
count,
self._attrs(action=related_action, record_type=record_type),
)
def record_completion(self, *, status: str, job_duration_seconds: float) -> None:
attributes = self._attrs(status=status)
self._add(self._job_runs_total, 1, attributes)
self._record(self._job_duration_seconds, job_duration_seconds, attributes)
class WorkflowRunCleanup:
def __init__(
self,
@@ -182,7 +29,6 @@ class WorkflowRunCleanup:
end_before: datetime.datetime | None = None,
workflow_run_repo: APIWorkflowRunRepository | None = None,
dry_run: bool = False,
task_label: str = "custom",
):
if (start_from is None) ^ (end_before is None):
raise ValueError("start_from and end_before must be both set or both omitted.")
@@ -200,11 +46,6 @@ class WorkflowRunCleanup:
self.batch_size = batch_size
self._cleanup_whitelist: set[str] | None = None
self.dry_run = dry_run
self._metrics = WorkflowRunCleanupMetrics(
dry_run=dry_run,
has_window=bool(start_from),
task_label=task_label,
)
self.free_plan_grace_period_days = dify_config.SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD
self.workflow_run_repo: APIWorkflowRunRepository
if workflow_run_repo:
@@ -233,193 +74,153 @@ class WorkflowRunCleanup:
related_totals = self._empty_related_counts() if self.dry_run else None
batch_index = 0
last_seen: tuple[datetime.datetime, str] | None = None
status = "success"
run_start = time.monotonic()
max_batch_interval_ms = dify_config.SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_MAX_INTERVAL
try:
while True:
batch_start = time.monotonic()
max_batch_interval_ms = int(os.environ.get("SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_MAX_INTERVAL", 200))
fetch_start = time.monotonic()
run_rows = self.workflow_run_repo.get_runs_batch_by_time_range(
start_from=self.window_start,
end_before=self.window_end,
last_seen=last_seen,
batch_size=self.batch_size,
while True:
batch_start = time.monotonic()
fetch_start = time.monotonic()
run_rows = self.workflow_run_repo.get_runs_batch_by_time_range(
start_from=self.window_start,
end_before=self.window_end,
last_seen=last_seen,
batch_size=self.batch_size,
)
if not run_rows:
logger.info("workflow_run_cleanup (batch #%s): no more rows to process", batch_index + 1)
break
batch_index += 1
last_seen = (run_rows[-1].created_at, run_rows[-1].id)
logger.info(
"workflow_run_cleanup (batch #%s): fetched %s rows in %sms",
batch_index,
len(run_rows),
int((time.monotonic() - fetch_start) * 1000),
)
tenant_ids = {row.tenant_id for row in run_rows}
filter_start = time.monotonic()
free_tenants = self._filter_free_tenants(tenant_ids)
logger.info(
"workflow_run_cleanup (batch #%s): filtered %s free tenants from %s tenants in %sms",
batch_index,
len(free_tenants),
len(tenant_ids),
int((time.monotonic() - filter_start) * 1000),
)
free_runs = [row for row in run_rows if row.tenant_id in free_tenants]
paid_or_skipped = len(run_rows) - len(free_runs)
if not free_runs:
skipped_message = (
f"[batch #{batch_index}] skipped (no sandbox runs in batch, {paid_or_skipped} paid/unknown)"
)
if not run_rows:
logger.info("workflow_run_cleanup (batch #%s): no more rows to process", batch_index + 1)
break
batch_index += 1
last_seen = (run_rows[-1].created_at, run_rows[-1].id)
logger.info(
"workflow_run_cleanup (batch #%s): fetched %s rows in %sms",
batch_index,
len(run_rows),
int((time.monotonic() - fetch_start) * 1000),
)
tenant_ids = {row.tenant_id for row in run_rows}
filter_start = time.monotonic()
free_tenants = self._filter_free_tenants(tenant_ids)
logger.info(
"workflow_run_cleanup (batch #%s): filtered %s free tenants from %s tenants in %sms",
batch_index,
len(free_tenants),
len(tenant_ids),
int((time.monotonic() - filter_start) * 1000),
)
free_runs = [row for row in run_rows if row.tenant_id in free_tenants]
paid_or_skipped = len(run_rows) - len(free_runs)
if not free_runs:
skipped_message = (
f"[batch #{batch_index}] skipped (no sandbox runs in batch, {paid_or_skipped} paid/unknown)"
)
click.echo(
click.style(
skipped_message,
fg="yellow",
)
)
self._metrics.record_batch(
batch_rows=len(run_rows),
targeted_runs=0,
skipped_runs=paid_or_skipped,
deleted_runs=0,
related_counts=None,
related_action=None,
batch_duration_seconds=time.monotonic() - batch_start,
)
continue
total_runs_targeted += len(free_runs)
if self.dry_run:
count_start = time.monotonic()
batch_counts = self.workflow_run_repo.count_runs_with_related(
free_runs,
count_node_executions=self._count_node_executions,
count_trigger_logs=self._count_trigger_logs,
)
logger.info(
"workflow_run_cleanup (batch #%s, dry_run): counted related records in %sms",
batch_index,
int((time.monotonic() - count_start) * 1000),
)
if related_totals is not None:
for key in related_totals:
related_totals[key] += batch_counts.get(key, 0)
sample_ids = ", ".join(run.id for run in free_runs[:5])
click.echo(
click.style(
f"[batch #{batch_index}] would delete {len(free_runs)} runs "
f"(sample ids: {sample_ids}) and skip {paid_or_skipped} paid/unknown",
fg="yellow",
)
)
logger.info(
"workflow_run_cleanup (batch #%s, dry_run): batch total %sms",
batch_index,
int((time.monotonic() - batch_start) * 1000),
)
self._metrics.record_batch(
batch_rows=len(run_rows),
targeted_runs=len(free_runs),
skipped_runs=paid_or_skipped,
deleted_runs=0,
related_counts={key: batch_counts.get(key, 0) for key in self._empty_related_counts()},
related_action="would_delete",
batch_duration_seconds=time.monotonic() - batch_start,
)
continue
try:
delete_start = time.monotonic()
counts = self.workflow_run_repo.delete_runs_with_related(
free_runs,
delete_node_executions=self._delete_node_executions,
delete_trigger_logs=self._delete_trigger_logs,
)
delete_ms = int((time.monotonic() - delete_start) * 1000)
except Exception:
logger.exception("Failed to delete workflow runs batch ending at %s", last_seen[0])
raise
total_runs_deleted += counts["runs"]
click.echo(
click.style(
f"[batch #{batch_index}] deleted runs: {counts['runs']} "
f"(nodes {counts['node_executions']}, offloads {counts['offloads']}, "
f"app_logs {counts['app_logs']}, trigger_logs {counts['trigger_logs']}, "
f"pauses {counts['pauses']}, pause_reasons {counts['pause_reasons']}); "
f"skipped {paid_or_skipped} paid/unknown",
fg="green",
skipped_message,
fg="yellow",
)
)
continue
total_runs_targeted += len(free_runs)
if self.dry_run:
count_start = time.monotonic()
batch_counts = self.workflow_run_repo.count_runs_with_related(
free_runs,
count_node_executions=self._count_node_executions,
count_trigger_logs=self._count_trigger_logs,
)
logger.info(
"workflow_run_cleanup (batch #%s, dry_run): counted related records in %sms",
batch_index,
int((time.monotonic() - count_start) * 1000),
)
if related_totals is not None:
for key in related_totals:
related_totals[key] += batch_counts.get(key, 0)
sample_ids = ", ".join(run.id for run in free_runs[:5])
click.echo(
click.style(
f"[batch #{batch_index}] would delete {len(free_runs)} runs "
f"(sample ids: {sample_ids}) and skip {paid_or_skipped} paid/unknown",
fg="yellow",
)
)
logger.info(
"workflow_run_cleanup (batch #%s): delete %sms, batch total %sms",
"workflow_run_cleanup (batch #%s, dry_run): batch total %sms",
batch_index,
delete_ms,
int((time.monotonic() - batch_start) * 1000),
)
self._metrics.record_batch(
batch_rows=len(run_rows),
targeted_runs=len(free_runs),
skipped_runs=paid_or_skipped,
deleted_runs=counts["runs"],
related_counts={key: counts.get(key, 0) for key in self._empty_related_counts()},
related_action="deleted",
batch_duration_seconds=time.monotonic() - batch_start,
continue
try:
delete_start = time.monotonic()
counts = self.workflow_run_repo.delete_runs_with_related(
free_runs,
delete_node_executions=self._delete_node_executions,
delete_trigger_logs=self._delete_trigger_logs,
)
delete_ms = int((time.monotonic() - delete_start) * 1000)
except Exception:
logger.exception("Failed to delete workflow runs batch ending at %s", last_seen[0])
raise
# Random sleep between batches to avoid overwhelming the database
sleep_ms = random.uniform(0, max_batch_interval_ms) # noqa: S311
logger.info("workflow_run_cleanup (batch #%s): sleeping for %.2fms", batch_index, sleep_ms)
time.sleep(sleep_ms / 1000)
if self.dry_run:
if self.window_start:
summary_message = (
f"Dry run complete. Would delete {total_runs_targeted} workflow runs "
f"between {self.window_start.isoformat()} and {self.window_end.isoformat()}"
)
else:
summary_message = (
f"Dry run complete. Would delete {total_runs_targeted} workflow runs "
f"before {self.window_end.isoformat()}"
)
if related_totals is not None:
summary_message = (
f"{summary_message}; related records: {self._format_related_counts(related_totals)}"
)
summary_color = "yellow"
else:
if self.window_start:
summary_message = (
f"Cleanup complete. Deleted {total_runs_deleted} workflow runs "
f"between {self.window_start.isoformat()} and {self.window_end.isoformat()}"
)
else:
summary_message = (
f"Cleanup complete. Deleted {total_runs_deleted} workflow runs "
f"before {self.window_end.isoformat()}"
)
summary_color = "white"
click.echo(click.style(summary_message, fg=summary_color))
except Exception:
status = "failed"
raise
finally:
self._metrics.record_completion(
status=status,
job_duration_seconds=time.monotonic() - run_start,
total_runs_deleted += counts["runs"]
click.echo(
click.style(
f"[batch #{batch_index}] deleted runs: {counts['runs']} "
f"(nodes {counts['node_executions']}, offloads {counts['offloads']}, "
f"app_logs {counts['app_logs']}, trigger_logs {counts['trigger_logs']}, "
f"pauses {counts['pauses']}, pause_reasons {counts['pause_reasons']}); "
f"skipped {paid_or_skipped} paid/unknown",
fg="green",
)
)
logger.info(
"workflow_run_cleanup (batch #%s): delete %sms, batch total %sms",
batch_index,
delete_ms,
int((time.monotonic() - batch_start) * 1000),
)
# Random sleep between batches to avoid overwhelming the database
sleep_ms = random.uniform(0, max_batch_interval_ms) # noqa: S311
logger.info("workflow_run_cleanup (batch #%s): sleeping for %.2fms", batch_index, sleep_ms)
time.sleep(sleep_ms / 1000)
if self.dry_run:
if self.window_start:
summary_message = (
f"Dry run complete. Would delete {total_runs_targeted} workflow runs "
f"between {self.window_start.isoformat()} and {self.window_end.isoformat()}"
)
else:
summary_message = (
f"Dry run complete. Would delete {total_runs_targeted} workflow runs "
f"before {self.window_end.isoformat()}"
)
if related_totals is not None:
summary_message = f"{summary_message}; related records: {self._format_related_counts(related_totals)}"
summary_color = "yellow"
else:
if self.window_start:
summary_message = (
f"Cleanup complete. Deleted {total_runs_deleted} workflow runs "
f"between {self.window_start.isoformat()} and {self.window_end.isoformat()}"
)
else:
summary_message = (
f"Cleanup complete. Deleted {total_runs_deleted} workflow runs before {self.window_end.isoformat()}"
)
summary_color = "white"
click.echo(click.style(summary_message, fg=summary_color))
def _filter_free_tenants(self, tenant_ids: Iterable[str]) -> set[str]:
tenant_id_list = list(tenant_ids)

View File

@@ -156,8 +156,7 @@ class VectorService:
)
# use full doc mode to generate segment's child chunk
processing_rule_dict = processing_rule.to_dict()
if processing_rule_dict["rules"] is not None:
processing_rule_dict["rules"]["parent_mode"] = ParentMode.FULL_DOC
processing_rule_dict["rules"]["parent_mode"] = ParentMode.FULL_DOC
documents = index_processor.transform(
[document],
embedding_model_instance=embedding_model_instance,

View File

@@ -46,7 +46,6 @@ def test_absolute_mode_calls_from_time_range():
end_before=end_before,
batch_size=200,
dry_run=True,
task_label="custom",
)
mock_from_days.assert_not_called()
@@ -75,7 +74,6 @@ def test_relative_mode_before_days_only_calls_from_days():
days=30,
batch_size=500,
dry_run=False,
task_label="before-30",
)
mock_from_time_range.assert_not_called()
@@ -107,7 +105,6 @@ def test_relative_mode_with_from_days_ago_calls_from_time_range():
end_before=fixed_now - datetime.timedelta(days=30),
batch_size=1000,
dry_run=False,
task_label="60to30",
)
mock_from_days.assert_not_called()

View File

@@ -4,7 +4,9 @@ from __future__ import annotations
import pytest
import dify_graph.graph_engine.response_coordinator.session as response_session_module
from dify_graph.enums import BuiltinNodeTypes, NodeExecutionType, NodeState, NodeType
from dify_graph.graph_engine.response_coordinator import RESPONSE_SESSION_NODE_TYPES
from dify_graph.graph_engine.response_coordinator.session import ResponseSession
from dify_graph.nodes.base.template import Template, TextSegment
@@ -33,14 +35,28 @@ class DummyNodeWithoutStreamingTemplate:
self.state = NodeState.UNKNOWN
def test_response_session_from_node_accepts_nodes_outside_previous_allowlist() -> None:
"""Session creation depends on the streaming-template contract rather than node type."""
def test_response_session_from_node_rejects_node_types_outside_allowlist() -> None:
"""Unsupported node types are rejected even if they expose a template."""
node = DummyResponseNode(
node_id="llm-node",
node_type=BuiltinNodeTypes.LLM,
template=Template(segments=[TextSegment(text="hello")]),
)
with pytest.raises(TypeError, match="RESPONSE_SESSION_NODE_TYPES"):
ResponseSession.from_node(node)
def test_response_session_from_node_supports_downstream_allowlist_extension(monkeypatch) -> None:
"""Downstream applications can extend the supported node-type list."""
node = DummyResponseNode(
node_id="llm-node",
node_type=BuiltinNodeTypes.LLM,
template=Template(segments=[TextSegment(text="hello")]),
)
extended_node_types = [*RESPONSE_SESSION_NODE_TYPES, BuiltinNodeTypes.LLM]
monkeypatch.setattr(response_session_module, "RESPONSE_SESSION_NODE_TYPES", extended_node_types)
session = ResponseSession.from_node(node)
assert session.node_id == "llm-node"

View File

@@ -1,4 +1,5 @@
import datetime
import os
from unittest.mock import MagicMock, patch
import pytest
@@ -281,6 +282,7 @@ class TestMessagesCleanService:
MessagesCleanService._batch_delete_message_relations(mock_db_session, ["msg1", "msg2"])
assert mock_db_session.execute.call_count == 8 # 8 tables to clean up
@patch.dict(os.environ, {"SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_MAX_INTERVAL": "500"})
def test_clean_messages_interval_from_env(self, mock_db_session, mock_policy):
service = MessagesCleanService(
policy=mock_policy,
@@ -299,13 +301,9 @@ class TestMessagesCleanService:
mock_db_session.execute.side_effect = mock_returns
mock_policy.filter_message_ids.return_value = ["msg1"]
with patch(
"services.retention.conversation.messages_clean_service.dify_config.SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_MAX_INTERVAL",
500,
):
with patch("services.retention.conversation.messages_clean_service.time.sleep") as mock_sleep:
with patch("services.retention.conversation.messages_clean_service.random.uniform") as mock_uniform:
mock_uniform.return_value = 300.0
service.run()
mock_uniform.assert_called_with(0, 500)
mock_sleep.assert_called_with(0.3)
with patch("services.retention.conversation.messages_clean_service.time.sleep") as mock_sleep:
with patch("services.retention.conversation.messages_clean_service.random.uniform") as mock_uniform:
mock_uniform.return_value = 300.0
service.run()
mock_uniform.assert_called_with(0, 500)
mock_sleep.assert_called_with(0.3)

View File

@@ -80,13 +80,7 @@ class TestWorkflowRunCleanupInit:
cfg.SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD = 0
cfg.BILLING_ENABLED = False
with pytest.raises(ValueError):
WorkflowRunCleanup(
days=30,
batch_size=10,
start_from=dt,
end_before=dt,
workflow_run_repo=mock_repo,
)
WorkflowRunCleanup(days=30, batch_size=10, start_from=dt, end_before=dt, workflow_run_repo=mock_repo)
def test_zero_batch_size_raises(self, mock_repo):
with patch("services.retention.workflow_run.clear_free_plan_expired_workflow_run_logs.dify_config") as cfg:
@@ -108,24 +102,10 @@ class TestWorkflowRunCleanupInit:
cfg.BILLING_ENABLED = False
start = datetime.datetime(2024, 1, 1)
end = datetime.datetime(2024, 6, 1)
c = WorkflowRunCleanup(
days=30,
batch_size=5,
start_from=start,
end_before=end,
workflow_run_repo=mock_repo,
)
c = WorkflowRunCleanup(days=30, batch_size=5, start_from=start, end_before=end, workflow_run_repo=mock_repo)
assert c.window_start == start
assert c.window_end == end
def test_default_task_label_is_custom(self, mock_repo):
with patch("services.retention.workflow_run.clear_free_plan_expired_workflow_run_logs.dify_config") as cfg:
cfg.SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD = 0
cfg.BILLING_ENABLED = False
c = WorkflowRunCleanup(days=30, batch_size=10, workflow_run_repo=mock_repo)
assert c._metrics._base_attributes["task_label"] == "custom"
# ---------------------------------------------------------------------------
# _empty_related_counts / _format_related_counts
@@ -413,12 +393,7 @@ class TestRunDryRunMode:
with patch("services.retention.workflow_run.clear_free_plan_expired_workflow_run_logs.dify_config") as cfg:
cfg.SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD = 0
cfg.BILLING_ENABLED = False
return WorkflowRunCleanup(
days=30,
batch_size=10,
workflow_run_repo=mock_repo,
dry_run=True,
)
return WorkflowRunCleanup(days=30, batch_size=10, workflow_run_repo=mock_repo, dry_run=True)
def test_dry_run_no_delete_called(self, mock_repo):
run = make_run("t1")

View File

@@ -265,61 +265,6 @@ def test_run_exits_on_empty_batch(monkeypatch: pytest.MonkeyPatch) -> None:
cleanup.run()
def test_run_records_metrics_on_success(monkeypatch: pytest.MonkeyPatch) -> None:
cutoff = datetime.datetime.now()
repo = FakeRepo(
batches=[[FakeRun("run-free", "t_free", cutoff)]],
delete_result={
"runs": 0,
"node_executions": 2,
"offloads": 1,
"app_logs": 3,
"trigger_logs": 4,
"pauses": 5,
"pause_reasons": 6,
},
)
cleanup = create_cleanup(monkeypatch, repo=repo, days=30, batch_size=10)
monkeypatch.setattr(cleanup_module.dify_config, "BILLING_ENABLED", False)
batch_calls: list[dict[str, object]] = []
completion_calls: list[dict[str, object]] = []
monkeypatch.setattr(cleanup._metrics, "record_batch", lambda **kwargs: batch_calls.append(kwargs))
monkeypatch.setattr(cleanup._metrics, "record_completion", lambda **kwargs: completion_calls.append(kwargs))
cleanup.run()
assert len(batch_calls) == 1
assert batch_calls[0]["batch_rows"] == 1
assert batch_calls[0]["targeted_runs"] == 1
assert batch_calls[0]["deleted_runs"] == 1
assert batch_calls[0]["related_action"] == "deleted"
assert len(completion_calls) == 1
assert completion_calls[0]["status"] == "success"
def test_run_records_failed_metrics(monkeypatch: pytest.MonkeyPatch) -> None:
class FailingRepo(FakeRepo):
def delete_runs_with_related(
self, runs: list[FakeRun], delete_node_executions=None, delete_trigger_logs=None
) -> dict[str, int]:
raise RuntimeError("delete failed")
cutoff = datetime.datetime.now()
repo = FailingRepo(batches=[[FakeRun("run-free", "t_free", cutoff)]])
cleanup = create_cleanup(monkeypatch, repo=repo, days=30, batch_size=10)
monkeypatch.setattr(cleanup_module.dify_config, "BILLING_ENABLED", False)
completion_calls: list[dict[str, object]] = []
monkeypatch.setattr(cleanup._metrics, "record_completion", lambda **kwargs: completion_calls.append(kwargs))
with pytest.raises(RuntimeError, match="delete failed"):
cleanup.run()
assert len(completion_calls) == 1
assert completion_calls[0]["status"] == "failed"
def test_run_dry_run_skips_deletions(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
cutoff = datetime.datetime.now()
repo = FakeRepo(

View File

@@ -540,20 +540,6 @@ class TestMessagesCleanServiceFromTimeRange:
assert service._batch_size == 1000 # default
assert service._dry_run is False # default
def test_explicit_task_label(self):
start_from = datetime.datetime(2024, 1, 1)
end_before = datetime.datetime(2024, 1, 2)
policy = BillingDisabledPolicy()
service = MessagesCleanService.from_time_range(
policy=policy,
start_from=start_from,
end_before=end_before,
task_label="60to30",
)
assert service._metrics._base_attributes["task_label"] == "60to30"
class TestMessagesCleanServiceFromDays:
"""Unit tests for MessagesCleanService.from_days factory method."""
@@ -633,54 +619,3 @@ class TestMessagesCleanServiceFromDays:
assert service._end_before == expected_end_before
assert service._batch_size == 1000 # default
assert service._dry_run is False # default
assert service._metrics._base_attributes["task_label"] == "custom"
class TestMessagesCleanServiceRun:
"""Unit tests for MessagesCleanService.run instrumentation behavior."""
def test_run_records_completion_metrics_on_success(self):
# Arrange
service = MessagesCleanService(
policy=BillingDisabledPolicy(),
start_from=datetime.datetime(2024, 1, 1),
end_before=datetime.datetime(2024, 1, 2),
batch_size=100,
dry_run=False,
)
expected_stats = {
"batches": 1,
"total_messages": 10,
"filtered_messages": 5,
"total_deleted": 5,
}
service._clean_messages_by_time_range = MagicMock(return_value=expected_stats) # type: ignore[method-assign]
completion_calls: list[dict[str, object]] = []
service._metrics.record_completion = lambda **kwargs: completion_calls.append(kwargs) # type: ignore[method-assign]
# Act
result = service.run()
# Assert
assert result == expected_stats
assert len(completion_calls) == 1
assert completion_calls[0]["status"] == "success"
def test_run_records_completion_metrics_on_failure(self):
# Arrange
service = MessagesCleanService(
policy=BillingDisabledPolicy(),
start_from=datetime.datetime(2024, 1, 1),
end_before=datetime.datetime(2024, 1, 2),
batch_size=100,
dry_run=False,
)
service._clean_messages_by_time_range = MagicMock(side_effect=RuntimeError("clean failed")) # type: ignore[method-assign]
completion_calls: list[dict[str, object]] = []
service._metrics.record_completion = lambda **kwargs: completion_calls.append(kwargs) # type: ignore[method-assign]
# Act & Assert
with pytest.raises(RuntimeError, match="clean failed"):
service.run()
assert len(completion_calls) == 1
assert completion_calls[0]["status"] == "failed"

6
api/uv.lock generated
View File

@@ -457,14 +457,14 @@ wheels = [
[[package]]
name = "authlib"
version = "1.6.9"
version = "1.6.7"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cryptography" },
]
sdist = { url = "https://files.pythonhosted.org/packages/af/98/00d3dd826d46959ad8e32af2dbb2398868fd9fd0683c26e56d0789bd0e68/authlib-1.6.9.tar.gz", hash = "sha256:d8f2421e7e5980cc1ddb4e32d3f5fa659cfaf60d8eaf3281ebed192e4ab74f04", size = 165134, upload-time = "2026-03-02T07:44:01.998Z" }
sdist = { url = "https://files.pythonhosted.org/packages/49/dc/ed1681bf1339dd6ea1ce56136bad4baabc6f7ad466e375810702b0237047/authlib-1.6.7.tar.gz", hash = "sha256:dbf10100011d1e1b34048c9d120e83f13b35d69a826ae762b93d2fb5aafc337b", size = 164950, upload-time = "2026-02-06T14:04:14.171Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/53/23/b65f568ed0c22f1efacb744d2db1a33c8068f384b8c9b482b52ebdbc3ef6/authlib-1.6.9-py2.py3-none-any.whl", hash = "sha256:f08b4c14e08f0861dc18a32357b33fbcfd2ea86cfe3fe149484b4d764c4a0ac3", size = 244197, upload-time = "2026-03-02T07:44:00.307Z" },
{ url = "https://files.pythonhosted.org/packages/f8/00/3ed12264094ec91f534fae429945efbaa9f8c666f3aa7061cc3b2a26a0cd/authlib-1.6.7-py2.py3-none-any.whl", hash = "sha256:c637340d9a02789d2efa1d003a7437d10d3e565237bcb5fcbc6c134c7b95bab0", size = 244115, upload-time = "2026-02-06T14:04:12.141Z" },
]
[[package]]

View File

@@ -12,7 +12,7 @@ NEXT_PUBLIC_API_PREFIX=http://localhost:5001/console/api
# console or api domain.
# example: http://udify.app/api
NEXT_PUBLIC_PUBLIC_API_PREFIX=http://localhost:5001/api
# Dev-only Hono proxy targets. Set the api prefixes above to https://localhost:5001/... to start the proxy with HTTPS.
# Dev-only Hono proxy targets. The frontend keeps requesting http://localhost:5001 directly.
HONO_PROXY_HOST=127.0.0.1
HONO_PROXY_PORT=5001
HONO_CONSOLE_API_PROXY_TARGET=

View File

@@ -161,40 +161,11 @@ describe('check-components-diff-coverage helpers', () => {
const coverage = getChangedBranchCoverage(entry, new Set([33]))
expect(coverage).toEqual({
covered: 0,
total: 1,
uncoveredBranches: [
{ armIndex: 0, line: 33 },
],
})
})
it('should require all branch arms when the branch condition changes', () => {
const entry = {
b: {
0: [0, 0],
},
branchMap: {
0: {
line: 30,
loc: { start: { line: 30 }, end: { line: 35 } },
locations: [
{ start: { line: 31 }, end: { line: 34 } },
{ start: { line: 35 }, end: { line: 38 } },
],
type: 'if',
},
},
}
const coverage = getChangedBranchCoverage(entry, new Set([30]))
expect(coverage).toEqual({
covered: 0,
total: 2,
uncoveredBranches: [
{ armIndex: 0, line: 31 },
{ armIndex: 0, line: 33 },
{ armIndex: 1, line: 35 },
],
})

View File

@@ -1,72 +0,0 @@
import {
getCoverageStats,
isRelevantTestFile,
isTrackedComponentSourceFile,
loadTrackedCoverageEntries,
} from '../scripts/components-coverage-common.mjs'
describe('components coverage common helpers', () => {
it('should identify tracked component source files and relevant tests', () => {
const excludedComponentCoverageFiles = new Set([
'web/app/components/share/types.ts',
])
expect(isTrackedComponentSourceFile('web/app/components/share/index.tsx', excludedComponentCoverageFiles)).toBe(true)
expect(isTrackedComponentSourceFile('web/app/components/share/types.ts', excludedComponentCoverageFiles)).toBe(false)
expect(isTrackedComponentSourceFile('web/app/components/provider/index.tsx', excludedComponentCoverageFiles)).toBe(false)
expect(isRelevantTestFile('web/__tests__/share/text-generation-run-once-flow.test.tsx')).toBe(true)
expect(isRelevantTestFile('web/app/components/share/__tests__/index.spec.tsx')).toBe(true)
expect(isRelevantTestFile('web/utils/format.spec.ts')).toBe(false)
})
it('should load only tracked coverage entries from mixed coverage paths', () => {
const context = {
excludedComponentCoverageFiles: new Set([
'web/app/components/share/types.ts',
]),
repoRoot: '/repo',
webRoot: '/repo/web',
}
const coverage = {
'/repo/web/app/components/provider/index.tsx': {
path: '/repo/web/app/components/provider/index.tsx',
statementMap: { 0: { start: { line: 1 }, end: { line: 1 } } },
s: { 0: 1 },
},
'app/components/share/index.tsx': {
path: 'app/components/share/index.tsx',
statementMap: { 0: { start: { line: 2 }, end: { line: 2 } } },
s: { 0: 1 },
},
'app/components/share/types.ts': {
path: 'app/components/share/types.ts',
statementMap: { 0: { start: { line: 3 }, end: { line: 3 } } },
s: { 0: 1 },
},
}
expect([...loadTrackedCoverageEntries(coverage, context).keys()]).toEqual([
'web/app/components/share/index.tsx',
])
})
it('should calculate coverage stats using statement-derived line hits', () => {
const entry = {
b: { 0: [1, 0] },
f: { 0: 1, 1: 0 },
s: { 0: 1, 1: 0 },
statementMap: {
0: { start: { line: 10 }, end: { line: 10 } },
1: { start: { line: 12 }, end: { line: 13 } },
},
}
expect(getCoverageStats(entry)).toEqual({
branches: { covered: 1, total: 2 },
functions: { covered: 1, total: 2 },
lines: { covered: 1, total: 2 },
statements: { covered: 1, total: 2 },
})
})
})

View File

@@ -137,31 +137,4 @@ describe('SelectDataSet', () => {
expect(screen.getByRole('link', { name: 'appDebug.feature.dataSet.toCreate' })).toHaveAttribute('href', '/datasets/create')
expect(screen.getByRole('button', { name: 'common.operation.add' })).toBeDisabled()
})
it('uses selectedIds as the initial modal selection', async () => {
const datasetOne = makeDataset({
id: 'set-1',
name: 'Dataset One',
})
mockUseInfiniteDatasets.mockReturnValue({
data: { pages: [{ data: [datasetOne] }] },
isLoading: false,
isFetchingNextPage: false,
fetchNextPage: vi.fn(),
hasNextPage: false,
})
const onSelect = vi.fn()
await act(async () => {
render(<SelectDataSet {...baseProps} onSelect={onSelect} selectedIds={['set-1']} />)
})
expect(screen.getByText('1 appDebug.feature.dataSet.selected')).toBeInTheDocument()
await act(async () => {
fireEvent.click(screen.getByRole('button', { name: 'common.operation.add' }))
})
expect(onSelect).toHaveBeenCalledWith([datasetOne])
})
})

View File

@@ -4,7 +4,7 @@ import type { DataSet } from '@/models/datasets'
import { useInfiniteScroll } from 'ahooks'
import Link from 'next/link'
import * as React from 'react'
import { useMemo, useRef, useState } from 'react'
import { useEffect, useMemo, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
import AppIcon from '@/app/components/base/app-icon'
import Badge from '@/app/components/base/badge'
@@ -31,21 +31,17 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
onSelect,
}) => {
const { t } = useTranslation()
const [selectedIdsInModal, setSelectedIdsInModal] = useState(() => selectedIds)
const [selected, setSelected] = useState<DataSet[]>([])
const canSelectMulti = true
const { formatIndexingTechniqueAndMethod } = useKnowledge()
const { data, isLoading, isFetchingNextPage, fetchNextPage, hasNextPage } = useInfiniteDatasets(
{ page: 1 },
{ enabled: isShow, staleTime: 0, refetchOnMount: 'always' },
)
const pages = data?.pages || []
const datasets = useMemo(() => {
const pages = data?.pages || []
return pages.flatMap(page => page.data.filter(item => item.indexing_technique || item.provider === 'external'))
}, [data])
const datasetMap = useMemo(() => new Map(datasets.map(item => [item.id, item])), [datasets])
const selected = useMemo(() => {
return selectedIdsInModal.map(id => datasetMap.get(id) || ({ id } as DataSet))
}, [datasetMap, selectedIdsInModal])
}, [pages])
const hasNoData = !isLoading && datasets.length === 0
const listRef = useRef<HTMLDivElement>(null)
@@ -65,14 +61,50 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
},
)
const toggleSelect = (dataSet: DataSet) => {
setSelectedIdsInModal((prev) => {
const isSelected = prev.includes(dataSet.id)
if (isSelected)
return prev.filter(id => id !== dataSet.id)
const prevSelectedIdsRef = useRef<string[]>([])
const hasUserModifiedSelectionRef = useRef(false)
useEffect(() => {
if (isShow)
hasUserModifiedSelectionRef.current = false
}, [isShow])
useEffect(() => {
const prevSelectedIds = prevSelectedIdsRef.current
const idsChanged = selectedIds.length !== prevSelectedIds.length
|| selectedIds.some((id, idx) => id !== prevSelectedIds[idx])
return canSelectMulti ? [...prev, dataSet.id] : [dataSet.id]
if (!selectedIds.length && (!hasUserModifiedSelectionRef.current || idsChanged)) {
setSelected([])
prevSelectedIdsRef.current = selectedIds
hasUserModifiedSelectionRef.current = false
return
}
if (!idsChanged && hasUserModifiedSelectionRef.current)
return
setSelected((prev) => {
const prevMap = new Map(prev.map(item => [item.id, item]))
const nextSelected = selectedIds
.map(id => datasets.find(item => item.id === id) || prevMap.get(id))
.filter(Boolean) as DataSet[]
return nextSelected
})
prevSelectedIdsRef.current = selectedIds
hasUserModifiedSelectionRef.current = false
}, [datasets, selectedIds])
const toggleSelect = (dataSet: DataSet) => {
hasUserModifiedSelectionRef.current = true
const isSelected = selected.some(item => item.id === dataSet.id)
if (isSelected) {
setSelected(selected.filter(item => item.id !== dataSet.id))
}
else {
if (canSelectMulti)
setSelected([...selected, dataSet])
else
setSelected([dataSet])
}
}
const handleSelect = () => {
@@ -94,7 +126,7 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
{hasNoData && (
<div
className="mt-6 flex h-[128px] items-center justify-center space-x-1 rounded-lg border text-[13px]"
className="mt-6 flex h-[128px] items-center justify-center space-x-1 rounded-lg border text-[13px]"
style={{
background: 'rgba(0, 0, 0, 0.02)',
borderColor: 'rgba(0, 0, 0, 0.02',
@@ -113,7 +145,7 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
key={item.id}
className={cn(
'flex h-10 cursor-pointer items-center rounded-lg border-[0.5px] border-components-panel-border-subtle bg-components-panel-on-panel-item-bg px-2 shadow-xs hover:border-components-panel-border hover:bg-components-panel-on-panel-item-bg-hover hover:shadow-sm',
selectedIdsInModal.includes(item.id) && 'border-[1.5px] border-components-option-card-option-selected-border bg-state-accent-hover shadow-xs hover:border-components-option-card-option-selected-border hover:bg-state-accent-hover hover:shadow-xs',
selected.some(i => i.id === item.id) && 'border-[1.5px] border-components-option-card-option-selected-border bg-state-accent-hover shadow-xs hover:border-components-option-card-option-selected-border hover:bg-state-accent-hover hover:shadow-xs',
!item.embedding_available && 'hover:border-components-panel-border-subtle hover:bg-components-panel-on-panel-item-bg hover:shadow-xs',
)}
onClick={() => {
@@ -163,7 +195,7 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
)}
{!isLoading && (
<div className="mt-8 flex items-center justify-between">
<div className="text-sm font-medium text-text-secondary">
<div className="text-sm font-medium text-text-secondary">
{selected.length > 0 && `${selected.length} ${t('feature.dataSet.selected', { ns: 'appDebug' })}`}
</div>
<div className="flex space-x-2">

View File

@@ -137,11 +137,5 @@ describe('ScoreThresholdItem', () => {
const input = screen.getByRole('textbox')
expect(input).toHaveValue('1')
})
it('should fall back to default value when value is undefined', () => {
render(<ScoreThresholdItem {...defaultProps} value={undefined} />)
const input = screen.getByRole('textbox')
expect(input).toHaveValue('0.7')
})
})
})

View File

@@ -6,7 +6,7 @@ import ParamItem from '.'
type Props = {
className?: string
value?: number
value: number
onChange: (key: string, value: number) => void
enable: boolean
hasSwitch?: boolean
@@ -20,18 +20,6 @@ const VALUE_LIMIT = {
max: 1,
}
const normalizeScoreThreshold = (value?: number): number => {
const normalizedValue = typeof value === 'number' && Number.isFinite(value)
? value
: VALUE_LIMIT.default
const roundedValue = Number.parseFloat(normalizedValue.toFixed(2))
return Math.min(
VALUE_LIMIT.max,
Math.max(VALUE_LIMIT.min, roundedValue),
)
}
const ScoreThresholdItem: FC<Props> = ({
className,
value,
@@ -41,10 +29,16 @@ const ScoreThresholdItem: FC<Props> = ({
onSwitchChange,
}) => {
const { t } = useTranslation()
const handleParamChange = (key: string, nextValue: number) => {
onChange(key, normalizeScoreThreshold(nextValue))
const handleParamChange = (key: string, value: number) => {
let notOutRangeValue = Number.parseFloat(value.toFixed(2))
notOutRangeValue = Math.max(VALUE_LIMIT.min, notOutRangeValue)
notOutRangeValue = Math.min(VALUE_LIMIT.max, notOutRangeValue)
onChange(key, notOutRangeValue)
}
const safeValue = normalizeScoreThreshold(value)
const safeValue = Math.min(
VALUE_LIMIT.max,
Math.max(VALUE_LIMIT.min, Number.parseFloat(value.toFixed(2))),
)
return (
<ParamItem

View File

@@ -203,7 +203,7 @@ const NodeSelector: FC<NodeSelectorProps> = ({
)
}
</PortalToFollowElemTrigger>
<PortalToFollowElemContent className="z-[1002]">
<PortalToFollowElemContent className="z-[1000]">
<div className={`rounded-lg border-[0.5px] border-components-panel-border bg-components-panel-bg shadow-lg ${popupClassName}`}>
<Tabs
tabs={tabs}

View File

@@ -169,7 +169,7 @@ const ToolPicker: FC<Props> = ({
{trigger}
</PortalToFollowElemTrigger>
<PortalToFollowElemContent className="z-[1002]">
<PortalToFollowElemContent className="z-[1000]">
<div className={cn('relative min-h-20 rounded-xl border-[0.5px] border-components-panel-border bg-components-panel-bg-blur shadow-lg backdrop-blur-sm', panelClassName)}>
<div className="p-2 pb-1">
<SearchBox

View File

@@ -117,7 +117,7 @@ const CodeEditor: FC<CodeEditorProps> = ({
<div className={cn('flex h-full flex-col overflow-hidden bg-components-input-bg-normal', hideTopMenu && 'pt-2', className)}>
{!hideTopMenu && (
<div className="flex items-center justify-between pl-2 pr-1 pt-1">
<div className="system-xs-semibold-uppercase py-0.5 text-text-secondary">
<div className="py-0.5 text-text-secondary system-xs-semibold-uppercase">
<span className="px-1 py-0.5">JSON</span>
</div>
<div className="flex items-center gap-x-0.5">

View File

@@ -928,6 +928,12 @@
"app/components/app/configuration/dataset-config/select-dataset/index.tsx": {
"no-restricted-imports": {
"count": 1
},
"react-hooks-extra/no-direct-set-state-in-use-effect": {
"count": 2
},
"tailwindcss/no-unnecessary-whitespace": {
"count": 2
}
},
"app/components/app/configuration/dataset-config/settings-modal/index.tsx": {
@@ -7875,9 +7881,6 @@
"no-restricted-imports": {
"count": 1
},
"tailwindcss/enforce-consistent-class-order": {
"count": 1
},
"ts/no-explicit-any": {
"count": 4
}

View File

@@ -210,7 +210,6 @@
"@types/sortablejs": "1.15.9",
"@typescript-eslint/parser": "8.57.0",
"@typescript/native-preview": "7.0.0-dev.20260312.1",
"@vitejs/plugin-basic-ssl": "2.2.0",
"@vitejs/plugin-react": "6.0.0",
"@vitejs/plugin-rsc": "0.5.21",
"@vitest/coverage-v8": "4.1.0",

View File

@@ -34,16 +34,7 @@ const toUpstreamCookieName = (cookieName: string) => {
return `__Host-${cookieName}`
}
const toLocalCookieName = (cookieName: string, options: LocalCookieRewriteOptions) => {
if (options.localSecure)
return cookieName
return cookieName.replace(SECURE_COOKIE_PREFIX_PATTERN, '')
}
type LocalCookieRewriteOptions = {
localSecure: boolean
}
const toLocalCookieName = (cookieName: string) => cookieName.replace(SECURE_COOKIE_PREFIX_PATTERN, '')
export const rewriteCookieHeaderForUpstream = (cookieHeader?: string) => {
if (!cookieHeader)
@@ -64,10 +55,7 @@ export const rewriteCookieHeaderForUpstream = (cookieHeader?: string) => {
.join('; ')
}
const rewriteSetCookieValueForLocal = (
setCookieValue: string,
options: LocalCookieRewriteOptions,
) => {
const rewriteSetCookieValueForLocal = (setCookieValue: string) => {
const [rawCookiePair, ...rawAttributes] = setCookieValue.split(';')
const separatorIndex = rawCookiePair.indexOf('=')
@@ -80,11 +68,11 @@ const rewriteSetCookieValueForLocal = (
.map(attribute => attribute.trim())
.filter(attribute =>
!COOKIE_DOMAIN_PATTERN.test(attribute)
&& (options.localSecure || !COOKIE_SECURE_PATTERN.test(attribute))
&& (options.localSecure || !COOKIE_PARTITIONED_PATTERN.test(attribute)),
&& !COOKIE_SECURE_PATTERN.test(attribute)
&& !COOKIE_PARTITIONED_PATTERN.test(attribute),
)
.map((attribute) => {
if (!options.localSecure && SAME_SITE_NONE_PATTERN.test(attribute))
if (SAME_SITE_NONE_PATTERN.test(attribute))
return 'SameSite=Lax'
if (COOKIE_PATH_PATTERN.test(attribute))
@@ -93,13 +81,10 @@ const rewriteSetCookieValueForLocal = (
return attribute
})
return [`${toLocalCookieName(cookieName, options)}=${cookieValue}`, ...rewrittenAttributes].join('; ')
return [`${toLocalCookieName(cookieName)}=${cookieValue}`, ...rewrittenAttributes].join('; ')
}
export const rewriteSetCookieHeadersForLocal = (
setCookieHeaders: string | string[] | undefined,
options: LocalCookieRewriteOptions,
): string[] | undefined => {
export const rewriteSetCookieHeadersForLocal = (setCookieHeaders?: string | string[]): string[] | undefined => {
if (!setCookieHeaders)
return undefined
@@ -107,7 +92,7 @@ export const rewriteSetCookieHeadersForLocal = (
? setCookieHeaders
: [setCookieHeaders]
return normalizedHeaders.map(setCookieValue => rewriteSetCookieValueForLocal(setCookieValue, options))
return normalizedHeaders.map(rewriteSetCookieValueForLocal)
}
export { DEFAULT_PROXY_TARGET }

View File

@@ -1,21 +0,0 @@
export type DevProxyProtocolEnv = Partial<Record<
| 'NEXT_PUBLIC_API_PREFIX'
| 'NEXT_PUBLIC_PUBLIC_API_PREFIX',
string
>>
const isHttpsUrl = (value?: string) => {
if (!value)
return false
try {
return new URL(value).protocol === 'https:'
}
catch {
return false
}
}
export const shouldUseHttpsForDevProxy = (env: DevProxyProtocolEnv = {}) => {
return isHttpsUrl(env.NEXT_PUBLIC_API_PREFIX) || isHttpsUrl(env.NEXT_PUBLIC_PUBLIC_API_PREFIX)
}

View File

@@ -1,5 +1,5 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { buildUpstreamUrl, createDevProxyApp, isAllowedDevOrigin, resolveDevProxyTargets, shouldUseHttpsForDevProxy } from './server'
import { buildUpstreamUrl, createDevProxyApp, isAllowedDevOrigin, resolveDevProxyTargets } from './server'
describe('dev proxy server', () => {
beforeEach(() => {
@@ -19,21 +19,6 @@ describe('dev proxy server', () => {
expect(targets.publicApiTarget).toBe('https://public.example.com')
})
// Scenario: the local dev proxy should switch to https when api prefixes are configured with https.
it('should enable https for the local dev proxy when api prefixes use https', () => {
// Assert
expect(shouldUseHttpsForDevProxy({
NEXT_PUBLIC_API_PREFIX: 'https://localhost:5001/console/api',
})).toBe(true)
expect(shouldUseHttpsForDevProxy({
NEXT_PUBLIC_PUBLIC_API_PREFIX: 'https://localhost:5001/api',
})).toBe(true)
expect(shouldUseHttpsForDevProxy({
NEXT_PUBLIC_API_PREFIX: 'http://localhost:5001/console/api',
NEXT_PUBLIC_PUBLIC_API_PREFIX: 'http://localhost:5001/api',
})).toBe(false)
})
// Scenario: target paths should not be duplicated when the incoming route already includes them.
it('should preserve prefixed targets when building upstream URLs', () => {
// Act
@@ -47,7 +32,6 @@ describe('dev proxy server', () => {
it('should only allow local development origins', () => {
// Assert
expect(isAllowedDevOrigin('http://localhost:3000')).toBe(true)
expect(isAllowedDevOrigin('https://localhost:3000')).toBe(true)
expect(isAllowedDevOrigin('http://127.0.0.1:3000')).toBe(true)
expect(isAllowedDevOrigin('https://example.com')).toBe(false)
})
@@ -102,39 +86,6 @@ describe('dev proxy server', () => {
])
})
// Scenario: secure local proxy responses should keep secure cross-site cookie attributes intact.
it('should preserve secure cookie attributes when the local proxy is https', async () => {
// Arrange
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValue(new Response('ok', {
status: 200,
headers: [
['set-cookie', '__Host-access_token=abc; Path=/console/api; Domain=cloud.dify.ai; Secure; SameSite=None; Partitioned'],
['set-cookie', '__Host-csrf_token=csrf; Path=/console/api; Domain=cloud.dify.ai; Secure; SameSite=None'],
],
}))
const app = createDevProxyApp({
consoleApiTarget: 'https://cloud.dify.ai',
publicApiTarget: 'https://public.dify.ai',
fetchImpl,
})
// Act
const response = await app.request('https://127.0.0.1:5001/console/api/apps?page=1', {
headers: {
Origin: 'https://localhost:3000',
Cookie: 'access_token=abc',
},
})
// Assert
expect(response.headers.getSetCookie()).toEqual([
'__Host-access_token=abc; Path=/; Secure; SameSite=None; Partitioned',
'__Host-csrf_token=csrf; Path=/; Secure; SameSite=None',
])
expect(response.headers.get('access-control-allow-origin')).toBe('https://localhost:3000')
expect(response.headers.get('access-control-allow-credentials')).toBe('true')
})
// Scenario: preflight requests should advertise allowed headers for credentialed cross-origin calls.
it('should answer CORS preflight requests', async () => {
// Arrange

View File

@@ -2,16 +2,10 @@ import type { Context, Hono } from 'hono'
import { Hono as HonoApp } from 'hono'
import { DEFAULT_PROXY_TARGET, rewriteCookieHeaderForUpstream, rewriteSetCookieHeadersForLocal } from './cookies'
export { shouldUseHttpsForDevProxy } from './protocol'
type DevProxyEnv = Partial<Record<
| 'HONO_CONSOLE_API_PROXY_TARGET'
| 'HONO_PUBLIC_API_PROXY_TARGET',
string
> & Record<
| 'NEXT_PUBLIC_API_PREFIX'
| 'NEXT_PUBLIC_PUBLIC_API_PREFIX',
string | undefined
>>
export type DevProxyTargets = {
@@ -99,15 +93,11 @@ const createProxyRequestHeaders = (request: Request, targetUrl: URL) => {
return headers
}
const createUpstreamResponseHeaders = (
response: Response,
requestOrigin: string | null | undefined,
localSecure: boolean,
) => {
const createUpstreamResponseHeaders = (response: Response, requestOrigin?: string | null) => {
const headers = new Headers(response.headers)
RESPONSE_HEADERS_TO_DROP.forEach(header => headers.delete(header))
const rewrittenSetCookies = rewriteSetCookieHeadersForLocal(response.headers.getSetCookie(), { localSecure })
const rewrittenSetCookies = rewriteSetCookieHeadersForLocal(response.headers.getSetCookie())
rewrittenSetCookies?.forEach((cookie) => {
headers.append('set-cookie', cookie)
})
@@ -136,11 +126,7 @@ const proxyRequest = async (
}
const upstreamResponse = await fetchImpl(targetUrl, requestInit)
const responseHeaders = createUpstreamResponseHeaders(
upstreamResponse,
context.req.header('origin'),
requestUrl.protocol === 'https:',
)
const responseHeaders = createUpstreamResponseHeaders(upstreamResponse, context.req.header('origin'))
return new Response(upstreamResponse.body, {
status: upstreamResponse.status,

13
web/pnpm-lock.yaml generated
View File

@@ -512,9 +512,6 @@ importers:
'@typescript/native-preview':
specifier: 7.0.0-dev.20260312.1
version: 7.0.0-dev.20260312.1
'@vitejs/plugin-basic-ssl':
specifier: 2.2.0
version: 2.2.0(@voidzero-dev/vite-plus-core@0.1.11(@types/node@25.5.0)(esbuild@0.27.2)(jiti@1.21.7)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.2))
'@vitejs/plugin-react':
specifier: 6.0.0
version: 6.0.0(@voidzero-dev/vite-plus-core@0.1.11(@types/node@25.5.0)(esbuild@0.27.2)(jiti@1.21.7)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.2))
@@ -3606,12 +3603,6 @@ packages:
resolution: {integrity: sha512-hBcWIOppZV14bi+eAmCZj8Elj8hVSUZJTpf1lgGBhVD85pervzQ1poM/qYfFUlPraYSZYP+ASg6To5BwYmUSGQ==}
engines: {node: '>=16'}
'@vitejs/plugin-basic-ssl@2.2.0':
resolution: {integrity: sha512-nmyQ1HGRkfUxjsv3jw0+hMhEdZdrtkvMTdkzRUaRWfiO6PCWw2V2Pz3gldCq96Tn9S8htcgdTxw/gmbLLEbfYw==}
engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0}
peerDependencies:
vite: ^6.0.0 || ^7.0.0 || ^8.0.0
'@vitejs/plugin-react@5.2.0':
resolution: {integrity: sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw==}
engines: {node: ^20.19.0 || >=22.12.0}
@@ -11039,10 +11030,6 @@ snapshots:
'@resvg/resvg-wasm': 2.4.0
satori: 0.16.0
'@vitejs/plugin-basic-ssl@2.2.0(@voidzero-dev/vite-plus-core@0.1.11(@types/node@25.5.0)(esbuild@0.27.2)(jiti@1.21.7)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.2))':
dependencies:
vite: '@voidzero-dev/vite-plus-core@0.1.11(@types/node@25.5.0)(esbuild@0.27.2)(jiti@1.21.7)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.2)'
'@vitejs/plugin-react@5.2.0(@voidzero-dev/vite-plus-core@0.1.11(@types/node@25.5.0)(esbuild@0.27.2)(jiti@1.21.7)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.2))':
dependencies:
'@babel/core': 7.29.0

View File

@@ -2,11 +2,6 @@ import fs from 'node:fs'
import path from 'node:path'
const DIFF_COVERAGE_IGNORE_LINE_TOKEN = 'diff-coverage-ignore-line:'
const DEFAULT_BRANCH_REF_CANDIDATES = ['origin/main', 'main']
export function normalizeDiffRangeMode(mode) {
return mode === 'exact' ? 'exact' : 'merge-base'
}
export function buildGitDiffRevisionArgs(base, head, mode = 'merge-base') {
return mode === 'exact'
@@ -14,46 +9,6 @@ export function buildGitDiffRevisionArgs(base, head, mode = 'merge-base') {
: [`${base}...${head}`]
}
export function resolveGitDiffContext({
base,
head,
mode = 'merge-base',
execGit,
}) {
const requestedMode = normalizeDiffRangeMode(mode)
const context = {
base,
head,
mode: requestedMode,
requestedMode,
reason: null,
useCombinedMergeDiff: false,
}
if (requestedMode !== 'exact' || !base || !head || !execGit)
return context
const baseCommit = resolveCommitSha(base, execGit) ?? base
const headCommit = resolveCommitSha(head, execGit) ?? head
const parents = getCommitParents(headCommit, execGit)
if (parents.length < 2)
return context
const [firstParent, secondParent] = parents
if (firstParent !== baseCommit)
return context
const defaultBranchRef = resolveDefaultBranchRef(execGit)
if (!defaultBranchRef || !isAncestor(secondParent, defaultBranchRef, execGit))
return context
return {
...context,
reason: `ignored merge from ${defaultBranchRef}`,
useCombinedMergeDiff: true,
}
}
export function parseChangedLineMap(diff, isTrackedComponentSourceFile) {
const lineMap = new Map()
let currentFile = null
@@ -67,7 +22,7 @@ export function parseChangedLineMap(diff, isTrackedComponentSourceFile) {
if (!currentFile || !isTrackedComponentSourceFile(currentFile))
continue
const match = line.match(/^@{2,}(?: -\d+(?:,\d+)?)+ \+(\d+)(?:,(\d+))? @{2,}/)
const match = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/)
if (!match)
continue
@@ -176,15 +131,14 @@ export function getChangedBranchCoverage(entry, changedLines) {
let total = 0
for (const [branchId, branch] of Object.entries(entry.branchMap ?? {})) {
if (!branchIntersectsChangedLines(branch, changedLines))
continue
const hits = Array.isArray(entry.b?.[branchId]) ? entry.b[branchId] : []
const locations = getBranchLocations(branch)
const armCount = Math.max(locations.length, hits.length)
const impactedArmIndexes = getImpactedBranchArmIndexes(branch, changedLines, armCount)
if (impactedArmIndexes.length === 0)
continue
for (const armIndex of impactedArmIndexes) {
for (let armIndex = 0; armIndex < armCount; armIndex += 1) {
total += 1
if ((hits[armIndex] ?? 0) > 0) {
covered += 1
@@ -265,99 +219,24 @@ function emptyIgnoreResult(changedLines = []) {
}
}
function getCommitParents(ref, execGit) {
const output = tryExecGit(execGit, ['rev-list', '--parents', '-n', '1', ref])
if (!output)
return []
return output
.trim()
.split(/\s+/)
.slice(1)
}
function resolveCommitSha(ref, execGit) {
return tryExecGit(execGit, ['rev-parse', '--verify', ref])?.trim() ?? null
}
function resolveDefaultBranchRef(execGit) {
const originHeadRef = tryExecGit(execGit, ['symbolic-ref', '--quiet', '--short', 'refs/remotes/origin/HEAD'])?.trim()
if (originHeadRef)
return originHeadRef
for (const ref of DEFAULT_BRANCH_REF_CANDIDATES) {
if (tryExecGit(execGit, ['rev-parse', '--verify', '-q', ref]))
return ref
}
return null
}
function isAncestor(ancestorRef, descendantRef, execGit) {
try {
execGit(['merge-base', '--is-ancestor', ancestorRef, descendantRef])
return true
}
catch {
function branchIntersectsChangedLines(branch, changedLines) {
if (!changedLines || changedLines.size === 0)
return false
}
}
function tryExecGit(execGit, args) {
try {
return execGit(args)
}
catch {
return null
}
if (rangeIntersectsChangedLines(branch.loc, changedLines))
return true
const locations = getBranchLocations(branch)
if (locations.some(location => rangeIntersectsChangedLines(location, changedLines)))
return true
return branch.line ? changedLines.has(branch.line) : false
}
function getBranchLocations(branch) {
return Array.isArray(branch?.locations) ? branch.locations.filter(Boolean) : []
}
function getImpactedBranchArmIndexes(branch, changedLines, armCount) {
if (!changedLines || changedLines.size === 0 || armCount === 0)
return []
const locations = getBranchLocations(branch)
if (isWholeBranchTouched(branch, changedLines, locations, armCount))
return Array.from({ length: armCount }, (_, armIndex) => armIndex)
const impactedArmIndexes = []
for (let armIndex = 0; armIndex < armCount; armIndex += 1) {
const location = locations[armIndex]
if (rangeIntersectsChangedLines(location, changedLines))
impactedArmIndexes.push(armIndex)
}
return impactedArmIndexes
}
function isWholeBranchTouched(branch, changedLines, locations, armCount) {
if (!changedLines || changedLines.size === 0)
return false
if (branch.line && changedLines.has(branch.line))
return true
const branchRange = branch.loc ?? branch
if (!rangeIntersectsChangedLines(branchRange, changedLines))
return false
if (locations.length === 0 || locations.length < armCount)
return true
for (const lineNumber of changedLines) {
if (!lineTouchesLocation(lineNumber, branchRange))
continue
if (!locations.some(location => lineTouchesLocation(lineNumber, location)))
return true
}
return false
}
function rangeIntersectsChangedLines(location, changedLines) {
if (!location || !changedLines || changedLines.size === 0)
return false
@@ -389,15 +268,6 @@ function getFirstChangedLineInRange(location, changedLines, fallbackLine = 1) {
return startLine ?? fallbackLine
}
function lineTouchesLocation(lineNumber, location) {
const startLine = getLocationStartLine(location)
const endLine = getLocationEndLine(location) ?? startLine
if (!startLine || !endLine)
return false
return lineNumber >= startLine && lineNumber <= endLine
}
function getLocationStartLine(location) {
return location?.start?.line ?? location?.line ?? null
}

View File

@@ -1,118 +0,0 @@
import { describe, expect, it, vi } from 'vitest'
import { parseChangedLineMap, resolveGitDiffContext } from './check-components-diff-coverage-lib.mjs'
function createExecGitMock(responses: Record<string, string | Error>) {
return vi.fn((args: string[]) => {
const key = args.join(' ')
const response = responses[key]
if (response instanceof Error)
throw response
if (response === undefined)
throw new Error(`Unexpected git args: ${key}`)
return response
})
}
describe('resolveGitDiffContext', () => {
it('switches exact diff to combined merge diff when head merges origin/main into the branch', () => {
const execGit = createExecGitMock({
'rev-parse --verify feature-parent-sha': 'feature-parent-sha\n',
'rev-parse --verify merge-sha': 'merge-sha\n',
'rev-list --parents -n 1 merge-sha': 'merge-sha feature-parent-sha main-parent-sha\n',
'symbolic-ref --quiet --short refs/remotes/origin/HEAD': 'origin/main\n',
'merge-base --is-ancestor main-parent-sha origin/main': '',
})
expect(resolveGitDiffContext({
base: 'feature-parent-sha',
head: 'merge-sha',
mode: 'exact',
execGit,
})).toEqual({
base: 'feature-parent-sha',
head: 'merge-sha',
mode: 'exact',
requestedMode: 'exact',
reason: 'ignored merge from origin/main',
useCombinedMergeDiff: true,
})
})
it('falls back to origin/main when origin/HEAD is unavailable', () => {
const execGit = createExecGitMock({
'rev-parse --verify feature-parent-sha': 'feature-parent-sha\n',
'rev-parse --verify merge-sha': 'merge-sha\n',
'rev-list --parents -n 1 merge-sha': 'merge-sha feature-parent-sha main-parent-sha\n',
'symbolic-ref --quiet --short refs/remotes/origin/HEAD': new Error('missing origin/HEAD'),
'rev-parse --verify -q origin/main': 'main-tip-sha\n',
'merge-base --is-ancestor main-parent-sha origin/main': '',
})
expect(resolveGitDiffContext({
base: 'feature-parent-sha',
head: 'merge-sha',
mode: 'exact',
execGit,
})).toEqual({
base: 'feature-parent-sha',
head: 'merge-sha',
mode: 'exact',
requestedMode: 'exact',
reason: 'ignored merge from origin/main',
useCombinedMergeDiff: true,
})
})
it('keeps exact diff when the second parent is not the default branch', () => {
const execGit = createExecGitMock({
'rev-parse --verify feature-parent-sha': 'feature-parent-sha\n',
'rev-parse --verify merge-sha': 'merge-sha\n',
'rev-list --parents -n 1 merge-sha': 'merge-sha feature-parent-sha topic-parent-sha\n',
'symbolic-ref --quiet --short refs/remotes/origin/HEAD': 'origin/main\n',
'merge-base --is-ancestor topic-parent-sha origin/main': new Error('not ancestor'),
})
expect(resolveGitDiffContext({
base: 'feature-parent-sha',
head: 'merge-sha',
mode: 'exact',
execGit,
})).toEqual({
base: 'feature-parent-sha',
head: 'merge-sha',
mode: 'exact',
requestedMode: 'exact',
reason: null,
useCombinedMergeDiff: false,
})
})
})
describe('parseChangedLineMap', () => {
it('parses regular diff hunks', () => {
const diff = [
'diff --git a/web/app/components/example.tsx b/web/app/components/example.tsx',
'+++ b/web/app/components/example.tsx',
'@@ -10,0 +11,2 @@',
].join('\n')
const changedLineMap = parseChangedLineMap(diff, () => true)
expect([...changedLineMap.get('web/app/components/example.tsx') ?? []]).toEqual([11, 12])
})
it('parses combined merge diff hunks', () => {
const diff = [
'diff --cc web/app/components/example.tsx',
'+++ b/web/app/components/example.tsx',
'@@@ -10,0 -10,0 +11,3 @@@',
].join('\n')
const changedLineMap = parseChangedLineMap(diff, () => true)
expect([...changedLineMap.get('web/app/components/example.tsx') ?? []]).toEqual([11, 12, 13])
})
})

View File

@@ -6,36 +6,41 @@ import {
getChangedBranchCoverage,
getChangedStatementCoverage,
getIgnoredChangedLinesFromFile,
normalizeDiffRangeMode,
getLineHits,
normalizeToRepoRelative,
parseChangedLineMap,
resolveGitDiffContext,
} from './check-components-diff-coverage-lib.mjs'
import { COMPONENT_COVERAGE_EXCLUDE_LABEL } from './component-coverage-filters.mjs'
import {
APP_COMPONENTS_PREFIX,
createComponentCoverageContext,
getModuleName,
isAnyComponentSourceFile,
isExcludedComponentSourceFile,
isTrackedComponentSourceFile,
loadTrackedCoverageEntries,
} from './components-coverage-common.mjs'
import { EXCLUDED_COMPONENT_MODULES } from './components-coverage-thresholds.mjs'
collectComponentCoverageExcludedFiles,
COMPONENT_COVERAGE_EXCLUDE_LABEL,
} from './component-coverage-filters.mjs'
import {
COMPONENTS_GLOBAL_THRESHOLDS,
EXCLUDED_COMPONENT_MODULES,
getComponentModuleThreshold,
} from './components-coverage-thresholds.mjs'
const REQUESTED_DIFF_RANGE_MODE = normalizeDiffRangeMode(process.env.DIFF_RANGE_MODE)
const APP_COMPONENTS_PREFIX = 'web/app/components/'
const APP_COMPONENTS_COVERAGE_PREFIX = 'app/components/'
const SHARED_TEST_PREFIX = 'web/__tests__/'
const STRICT_TEST_FILE_TOUCH = process.env.STRICT_COMPONENT_TEST_TOUCH === 'true'
const EXCLUDED_MODULES_LABEL = [...EXCLUDED_COMPONENT_MODULES].sort().join(', ')
const DIFF_RANGE_MODE = process.env.DIFF_RANGE_MODE === 'exact' ? 'exact' : 'merge-base'
const repoRoot = repoRootFromCwd()
const context = createComponentCoverageContext(repoRoot)
const webRoot = path.join(repoRoot, 'web')
const excludedComponentCoverageFiles = new Set(
collectComponentCoverageExcludedFiles(path.join(webRoot, 'app/components'), { pathPrefix: 'web/app/components' }),
)
const baseSha = process.env.BASE_SHA?.trim()
const headSha = process.env.HEAD_SHA?.trim() || 'HEAD'
const coverageFinalPath = path.join(context.webRoot, 'coverage', 'coverage-final.json')
const coverageFinalPath = path.join(webRoot, 'coverage', 'coverage-final.json')
if (!baseSha || /^0+$/.test(baseSha)) {
appendSummary([
'### app/components Pure Diff Coverage',
'### app/components Diff Coverage',
'',
'Skipped pure diff coverage check because `BASE_SHA` was not available.',
'Skipped diff coverage check because `BASE_SHA` was not available.',
])
process.exit(0)
}
@@ -45,36 +50,55 @@ if (!fs.existsSync(coverageFinalPath)) {
process.exit(1)
}
const diffContext = resolveGitDiffContext({
base: baseSha,
head: headSha,
mode: REQUESTED_DIFF_RANGE_MODE,
execGit,
})
const coverage = JSON.parse(fs.readFileSync(coverageFinalPath, 'utf8'))
const changedFiles = getChangedFiles(diffContext)
const changedFiles = getChangedFiles(baseSha, headSha)
const changedComponentSourceFiles = changedFiles.filter(isAnyComponentSourceFile)
const changedSourceFiles = changedComponentSourceFiles.filter(filePath => isTrackedComponentSourceFile(filePath, context.excludedComponentCoverageFiles))
const changedExcludedSourceFiles = changedComponentSourceFiles.filter(filePath => isExcludedComponentSourceFile(filePath, context.excludedComponentCoverageFiles))
const changedSourceFiles = changedComponentSourceFiles.filter(isTrackedComponentSourceFile)
const changedExcludedSourceFiles = changedComponentSourceFiles.filter(isExcludedComponentSourceFile)
const changedTestFiles = changedFiles.filter(isRelevantTestFile)
if (changedSourceFiles.length === 0) {
appendSummary(buildSkipSummary(changedExcludedSourceFiles))
process.exit(0)
}
const coverageEntries = loadTrackedCoverageEntries(coverage, context)
const diffChanges = getChangedLineMap(diffContext)
const coverageEntries = new Map()
for (const [file, entry] of Object.entries(coverage)) {
const repoRelativePath = normalizeToRepoRelative(entry.path ?? file, {
appComponentsCoveragePrefix: APP_COMPONENTS_COVERAGE_PREFIX,
appComponentsPrefix: APP_COMPONENTS_PREFIX,
repoRoot,
sharedTestPrefix: SHARED_TEST_PREFIX,
webRoot,
})
if (!isTrackedComponentSourceFile(repoRelativePath))
continue
coverageEntries.set(repoRelativePath, entry)
}
const fileCoverageRows = []
const moduleCoverageMap = new Map()
for (const [file, entry] of coverageEntries.entries()) {
const stats = getCoverageStats(entry)
const moduleName = getModuleName(file)
fileCoverageRows.push({ file, moduleName, ...stats })
mergeCoverageStats(moduleCoverageMap, moduleName, stats)
}
const overallCoverage = sumCoverageStats(fileCoverageRows)
const diffChanges = getChangedLineMap(baseSha, headSha)
const diffRows = []
const ignoredDiffLines = []
const invalidIgnorePragmas = []
for (const [file, changedLines] of diffChanges.entries()) {
if (!isTrackedComponentSourceFile(file, context.excludedComponentCoverageFiles))
if (!isTrackedComponentSourceFile(file))
continue
const entry = coverageEntries.get(file)
const ignoreInfo = getIgnoredChangedLinesFromFile(path.join(repoRoot, file), changedLines)
for (const [line, reason] of ignoreInfo.ignoredLines.entries()) {
ignoredDiffLines.push({
file,
@@ -82,7 +106,6 @@ for (const [file, changedLines] of diffChanges.entries()) {
reason,
})
}
for (const invalidPragma of ignoreInfo.invalidPragmas) {
invalidIgnorePragmas.push({
file,
@@ -114,16 +137,40 @@ const diffTotals = diffRows.reduce((acc, row) => {
const diffStatementFailures = diffRows.filter(row => row.statements.uncoveredLines.length > 0)
const diffBranchFailures = diffRows.filter(row => row.branches.uncoveredBranches.length > 0)
const overallThresholdFailures = getThresholdFailures(overallCoverage, COMPONENTS_GLOBAL_THRESHOLDS)
const moduleCoverageRows = [...moduleCoverageMap.entries()]
.map(([moduleName, stats]) => ({
moduleName,
stats,
thresholds: getComponentModuleThreshold(moduleName),
}))
.map(row => ({
...row,
failures: row.thresholds ? getThresholdFailures(row.stats, row.thresholds) : [],
}))
const moduleThresholdFailures = moduleCoverageRows
.filter(row => row.failures.length > 0)
.flatMap(row => row.failures.map(failure => ({
moduleName: row.moduleName,
...failure,
})))
const hasRelevantTestChanges = changedTestFiles.length > 0
const missingTestTouch = !hasRelevantTestChanges
appendSummary(buildSummary({
changedSourceFiles,
diffContext,
overallCoverage,
overallThresholdFailures,
moduleCoverageRows,
moduleThresholdFailures,
diffBranchFailures,
diffRows,
diffStatementFailures,
diffTotals,
changedSourceFiles,
changedTestFiles,
ignoredDiffLines,
invalidIgnorePragmas,
missingTestTouch,
}))
if (process.env.CI) {
@@ -131,51 +178,107 @@ if (process.env.CI) {
const firstLine = failure.statements.uncoveredLines[0] ?? 1
console.log(`::error file=${failure.file},line=${firstLine}::Uncovered changed statements: ${formatLineRanges(failure.statements.uncoveredLines)}`)
}
for (const failure of diffBranchFailures.slice(0, 20)) {
const firstBranch = failure.branches.uncoveredBranches[0]
const line = firstBranch?.line ?? 1
console.log(`::error file=${failure.file},line=${line}::Uncovered changed branches: ${formatBranchRefs(failure.branches.uncoveredBranches)}`)
}
for (const invalidPragma of invalidIgnorePragmas.slice(0, 20)) {
console.log(`::error file=${invalidPragma.file},line=${invalidPragma.line}::Invalid diff coverage ignore pragma: ${invalidPragma.reason}`)
}
}
if (
diffStatementFailures.length > 0
overallThresholdFailures.length > 0
|| moduleThresholdFailures.length > 0
|| diffStatementFailures.length > 0
|| diffBranchFailures.length > 0
|| invalidIgnorePragmas.length > 0
|| (STRICT_TEST_FILE_TOUCH && missingTestTouch)
) {
process.exit(1)
}
function buildSummary({
changedSourceFiles,
diffContext,
overallCoverage,
overallThresholdFailures,
moduleCoverageRows,
moduleThresholdFailures,
diffBranchFailures,
diffRows,
diffStatementFailures,
diffTotals,
changedSourceFiles,
changedTestFiles,
ignoredDiffLines,
invalidIgnorePragmas,
missingTestTouch,
}) {
const lines = [
'### app/components Pure Diff Coverage',
'### app/components Diff Coverage',
'',
...buildDiffContextSummary(diffContext),
`Compared \`${baseSha.slice(0, 12)}\` -> \`${headSha.slice(0, 12)}\``,
`Diff range mode: \`${DIFF_RANGE_MODE}\``,
'',
`Excluded modules: \`${EXCLUDED_MODULES_LABEL}\``,
`Excluded file kinds: \`${COMPONENT_COVERAGE_EXCLUDE_LABEL}\``,
'',
'| Check | Result | Details |',
'|---|---:|---|',
`| Overall tracked lines | ${formatPercent(overallCoverage.lines)} | ${overallCoverage.lines.covered}/${overallCoverage.lines.total}; threshold ${COMPONENTS_GLOBAL_THRESHOLDS.lines}% |`,
`| Overall tracked statements | ${formatPercent(overallCoverage.statements)} | ${overallCoverage.statements.covered}/${overallCoverage.statements.total}; threshold ${COMPONENTS_GLOBAL_THRESHOLDS.statements}% |`,
`| Overall tracked functions | ${formatPercent(overallCoverage.functions)} | ${overallCoverage.functions.covered}/${overallCoverage.functions.total}; threshold ${COMPONENTS_GLOBAL_THRESHOLDS.functions}% |`,
`| Overall tracked branches | ${formatPercent(overallCoverage.branches)} | ${overallCoverage.branches.covered}/${overallCoverage.branches.total}; threshold ${COMPONENTS_GLOBAL_THRESHOLDS.branches}% |`,
`| Changed statements | ${formatDiffPercent(diffTotals.statements)} | ${diffTotals.statements.covered}/${diffTotals.statements.total} |`,
`| Changed branches | ${formatDiffPercent(diffTotals.branches)} | ${diffTotals.branches.covered}/${diffTotals.branches.total} |`,
'',
]
if (overallThresholdFailures.length > 0) {
lines.push('Overall thresholds failed:')
for (const failure of overallThresholdFailures)
lines.push(`- ${failure.metric}: ${failure.actual.toFixed(2)}% < ${failure.expected}%`)
lines.push('')
}
if (moduleThresholdFailures.length > 0) {
lines.push('Module thresholds failed:')
for (const failure of moduleThresholdFailures)
lines.push(`- ${failure.moduleName} ${failure.metric}: ${failure.actual.toFixed(2)}% < ${failure.expected}%`)
lines.push('')
}
const moduleRows = moduleCoverageRows
.map(({ moduleName, stats, thresholds, failures }) => ({
moduleName,
lines: percentage(stats.lines.covered, stats.lines.total),
statements: percentage(stats.statements.covered, stats.statements.total),
functions: percentage(stats.functions.covered, stats.functions.total),
branches: percentage(stats.branches.covered, stats.branches.total),
thresholds,
failures,
}))
.sort((a, b) => {
if (a.failures.length !== b.failures.length)
return b.failures.length - a.failures.length
return a.lines - b.lines || a.moduleName.localeCompare(b.moduleName)
})
lines.push('<details><summary>Module coverage</summary>')
lines.push('')
lines.push('| Module | Lines | Statements | Functions | Branches | Thresholds | Status |')
lines.push('|---|---:|---:|---:|---:|---|---|')
for (const row of moduleRows) {
const thresholdLabel = row.thresholds
? `L${row.thresholds.lines}/S${row.thresholds.statements}/F${row.thresholds.functions}/B${row.thresholds.branches}`
: 'n/a'
const status = row.thresholds ? (row.failures.length > 0 ? 'fail' : 'pass') : 'info'
lines.push(`| ${row.moduleName} | ${row.lines.toFixed(2)}% | ${row.statements.toFixed(2)}% | ${row.functions.toFixed(2)}% | ${row.branches.toFixed(2)}% | ${thresholdLabel} | ${status} |`)
}
lines.push('</details>')
lines.push('')
const changedRows = diffRows
.filter(row => row.statements.total > 0 || row.branches.total > 0)
.sort((a, b) => {
@@ -194,45 +297,59 @@ function buildSummary({
lines.push('</details>')
lines.push('')
if (missingTestTouch) {
lines.push(`Warning: tracked source files changed under \`web/app/components/\`, but no test files changed under \`web/app/components/**\` or \`web/__tests__/\`.`)
if (STRICT_TEST_FILE_TOUCH)
lines.push('`STRICT_COMPONENT_TEST_TOUCH=true` is enabled, so this warning fails the check.')
lines.push('')
}
else {
lines.push(`Relevant test files changed: ${changedTestFiles.length}`)
lines.push('')
}
if (diffStatementFailures.length > 0) {
lines.push('Uncovered changed statements:')
for (const row of diffStatementFailures)
for (const row of diffStatementFailures) {
lines.push(`- ${row.file.replace('web/', '')}: ${formatLineRanges(row.statements.uncoveredLines)}`)
}
lines.push('')
}
if (diffBranchFailures.length > 0) {
lines.push('Uncovered changed branches:')
for (const row of diffBranchFailures)
for (const row of diffBranchFailures) {
lines.push(`- ${row.file.replace('web/', '')}: ${formatBranchRefs(row.branches.uncoveredBranches)}`)
}
lines.push('')
}
if (ignoredDiffLines.length > 0) {
lines.push('Ignored changed lines via pragma:')
for (const ignoredLine of ignoredDiffLines)
for (const ignoredLine of ignoredDiffLines) {
lines.push(`- ${ignoredLine.file.replace('web/', '')}:${ignoredLine.line} - ${ignoredLine.reason}`)
}
lines.push('')
}
if (invalidIgnorePragmas.length > 0) {
lines.push('Invalid diff coverage ignore pragmas:')
for (const invalidPragma of invalidIgnorePragmas)
for (const invalidPragma of invalidIgnorePragmas) {
lines.push(`- ${invalidPragma.file.replace('web/', '')}:${invalidPragma.line} - ${invalidPragma.reason}`)
}
lines.push('')
}
lines.push(`Changed source files checked: ${changedSourceFiles.length}`)
lines.push('Blocking rules: uncovered changed statements, uncovered changed branches, invalid ignore pragmas.')
lines.push(`Changed statement coverage: ${formatDiffPercent(diffTotals.statements)}`)
lines.push(`Changed branch coverage: ${formatDiffPercent(diffTotals.branches)}`)
return lines
}
function buildSkipSummary(changedExcludedSourceFiles) {
const lines = [
'### app/components Pure Diff Coverage',
'',
...buildDiffContextSummary(diffContext),
'### app/components Diff Coverage',
'',
`Excluded modules: \`${EXCLUDED_MODULES_LABEL}\``,
`Excluded file kinds: \`${COMPONENT_COVERAGE_EXCLUDE_LABEL}\``,
@@ -240,60 +357,146 @@ function buildSkipSummary(changedExcludedSourceFiles) {
]
if (changedExcludedSourceFiles.length > 0) {
lines.push('Only excluded component modules or type-only files changed, so pure diff coverage was skipped.')
lines.push('Only excluded component modules or type-only files changed, so diff coverage check was skipped.')
lines.push(`Skipped files: ${changedExcludedSourceFiles.length}`)
}
else {
lines.push('No tracked source changes under `web/app/components/`. Pure diff coverage skipped.')
lines.push('No source changes under tracked `web/app/components/`. Diff coverage check skipped.')
}
return lines
}
function buildDiffContextSummary(diffContext) {
const lines = [
`Compared \`${diffContext.base.slice(0, 12)}\` -> \`${diffContext.head.slice(0, 12)}\``,
]
if (diffContext.useCombinedMergeDiff) {
lines.push(`Requested diff range mode: \`${diffContext.requestedMode}\``)
lines.push(`Effective diff strategy: \`combined-merge\` (${diffContext.reason})`)
}
else if (diffContext.reason) {
lines.push(`Requested diff range mode: \`${diffContext.requestedMode}\``)
lines.push(`Effective diff range mode: \`${diffContext.mode}\` (${diffContext.reason})`)
}
else {
lines.push(`Diff range mode: \`${diffContext.mode}\``)
}
return lines
}
function getChangedFiles(diffContext) {
if (diffContext.useCombinedMergeDiff) {
const output = execGit(['diff-tree', '--cc', '--no-commit-id', '--name-only', '-r', diffContext.head, '--', APP_COMPONENTS_PREFIX])
return output
.split('\n')
.map(line => line.trim())
.filter(Boolean)
}
const output = execGit(['diff', '--name-only', '--diff-filter=ACMR', ...buildGitDiffRevisionArgs(diffContext.base, diffContext.head, diffContext.mode), '--', APP_COMPONENTS_PREFIX])
function getChangedFiles(base, head) {
const output = execGit(['diff', '--name-only', '--diff-filter=ACMR', ...buildGitDiffRevisionArgs(base, head, DIFF_RANGE_MODE), '--', 'web/app/components', 'web/__tests__'])
return output
.split('\n')
.map(line => line.trim())
.filter(Boolean)
}
function getChangedLineMap(diffContext) {
if (diffContext.useCombinedMergeDiff) {
const diff = execGit(['diff-tree', '--cc', '--no-commit-id', '-r', '--unified=0', diffContext.head, '--', APP_COMPONENTS_PREFIX])
return parseChangedLineMap(diff, filePath => isTrackedComponentSourceFile(filePath, context.excludedComponentCoverageFiles))
}
function getChangedLineMap(base, head) {
const diff = execGit(['diff', '--unified=0', '--no-color', '--diff-filter=ACMR', ...buildGitDiffRevisionArgs(base, head, DIFF_RANGE_MODE), '--', 'web/app/components'])
return parseChangedLineMap(diff, isTrackedComponentSourceFile)
}
const diff = execGit(['diff', '--unified=0', '--no-color', '--diff-filter=ACMR', ...buildGitDiffRevisionArgs(diffContext.base, diffContext.head, diffContext.mode), '--', APP_COMPONENTS_PREFIX])
return parseChangedLineMap(diff, filePath => isTrackedComponentSourceFile(filePath, context.excludedComponentCoverageFiles))
function isAnyComponentSourceFile(filePath) {
return filePath.startsWith(APP_COMPONENTS_PREFIX)
&& /\.(?:ts|tsx)$/.test(filePath)
&& !isTestLikePath(filePath)
}
function isTrackedComponentSourceFile(filePath) {
return isAnyComponentSourceFile(filePath)
&& !isExcludedComponentSourceFile(filePath)
}
function isExcludedComponentSourceFile(filePath) {
return isAnyComponentSourceFile(filePath)
&& (
EXCLUDED_COMPONENT_MODULES.has(getModuleName(filePath))
|| excludedComponentCoverageFiles.has(filePath)
)
}
function isRelevantTestFile(filePath) {
return filePath.startsWith(SHARED_TEST_PREFIX)
|| (filePath.startsWith(APP_COMPONENTS_PREFIX) && isTestLikePath(filePath) && !isExcludedComponentTestFile(filePath))
}
function isExcludedComponentTestFile(filePath) {
if (!filePath.startsWith(APP_COMPONENTS_PREFIX))
return false
return EXCLUDED_COMPONENT_MODULES.has(getModuleName(filePath))
}
function isTestLikePath(filePath) {
return /(?:^|\/)__tests__\//.test(filePath)
|| /(?:^|\/)__mocks__\//.test(filePath)
|| /\.(?:spec|test)\.(?:ts|tsx)$/.test(filePath)
|| /\.stories\.(?:ts|tsx)$/.test(filePath)
|| /\.d\.ts$/.test(filePath)
}
function getCoverageStats(entry) {
const lineHits = getLineHits(entry)
const statementHits = Object.values(entry.s ?? {})
const functionHits = Object.values(entry.f ?? {})
const branchHits = Object.values(entry.b ?? {}).flat()
return {
lines: {
covered: Object.values(lineHits).filter(count => count > 0).length,
total: Object.keys(lineHits).length,
},
statements: {
covered: statementHits.filter(count => count > 0).length,
total: statementHits.length,
},
functions: {
covered: functionHits.filter(count => count > 0).length,
total: functionHits.length,
},
branches: {
covered: branchHits.filter(count => count > 0).length,
total: branchHits.length,
},
}
}
function sumCoverageStats(rows) {
const total = createEmptyCoverageStats()
for (const row of rows)
addCoverageStats(total, row)
return total
}
function mergeCoverageStats(map, moduleName, stats) {
const existing = map.get(moduleName) ?? createEmptyCoverageStats()
addCoverageStats(existing, stats)
map.set(moduleName, existing)
}
function addCoverageStats(target, source) {
for (const metric of ['lines', 'statements', 'functions', 'branches']) {
target[metric].covered += source[metric].covered
target[metric].total += source[metric].total
}
}
function createEmptyCoverageStats() {
return {
lines: { covered: 0, total: 0 },
statements: { covered: 0, total: 0 },
functions: { covered: 0, total: 0 },
branches: { covered: 0, total: 0 },
}
}
function getThresholdFailures(stats, thresholds) {
const failures = []
for (const metric of ['lines', 'statements', 'functions', 'branches']) {
const actual = percentage(stats[metric].covered, stats[metric].total)
const expected = thresholds[metric]
if (actual < expected) {
failures.push({
metric,
actual,
expected,
})
}
}
return failures
}
function getModuleName(filePath) {
const relativePath = filePath.slice(APP_COMPONENTS_PREFIX.length)
if (!relativePath)
return '(root)'
const segments = relativePath.split('/')
return segments.length === 1 ? '(root)' : segments[0]
}
function formatLineRanges(lines) {
@@ -333,6 +536,10 @@ function percentage(covered, total) {
return (covered / total) * 100
}
function formatPercent(metric) {
return `${percentage(metric.covered, metric.total).toFixed(2)}%`
}
function formatDiffPercent(metric) {
if (metric.total === 0)
return 'n/a'

View File

@@ -1,195 +0,0 @@
import fs from 'node:fs'
import path from 'node:path'
import { getLineHits, normalizeToRepoRelative } from './check-components-diff-coverage-lib.mjs'
import { collectComponentCoverageExcludedFiles } from './component-coverage-filters.mjs'
import { EXCLUDED_COMPONENT_MODULES } from './components-coverage-thresholds.mjs'
export const APP_COMPONENTS_ROOT = 'web/app/components'
export const APP_COMPONENTS_PREFIX = `${APP_COMPONENTS_ROOT}/`
export const APP_COMPONENTS_COVERAGE_PREFIX = 'app/components/'
export const SHARED_TEST_PREFIX = 'web/__tests__/'
export function createComponentCoverageContext(repoRoot) {
const webRoot = path.join(repoRoot, 'web')
const excludedComponentCoverageFiles = new Set(
collectComponentCoverageExcludedFiles(path.join(webRoot, 'app/components'), { pathPrefix: APP_COMPONENTS_ROOT }),
)
return {
excludedComponentCoverageFiles,
repoRoot,
webRoot,
}
}
export function loadTrackedCoverageEntries(coverage, context) {
const coverageEntries = new Map()
for (const [file, entry] of Object.entries(coverage)) {
const repoRelativePath = normalizeToRepoRelative(entry.path ?? file, {
appComponentsCoveragePrefix: APP_COMPONENTS_COVERAGE_PREFIX,
appComponentsPrefix: APP_COMPONENTS_PREFIX,
repoRoot: context.repoRoot,
sharedTestPrefix: SHARED_TEST_PREFIX,
webRoot: context.webRoot,
})
if (!isTrackedComponentSourceFile(repoRelativePath, context.excludedComponentCoverageFiles))
continue
coverageEntries.set(repoRelativePath, entry)
}
return coverageEntries
}
export function collectTrackedComponentSourceFiles(context) {
const trackedFiles = []
walkComponentSourceFiles(path.join(context.webRoot, 'app/components'), (absolutePath) => {
const repoRelativePath = path.relative(context.repoRoot, absolutePath).split(path.sep).join('/')
if (isTrackedComponentSourceFile(repoRelativePath, context.excludedComponentCoverageFiles))
trackedFiles.push(repoRelativePath)
})
trackedFiles.sort((a, b) => a.localeCompare(b))
return trackedFiles
}
export function isTestLikePath(filePath) {
return /(?:^|\/)__tests__\//.test(filePath)
|| /(?:^|\/)__mocks__\//.test(filePath)
|| /\.(?:spec|test)\.(?:ts|tsx)$/.test(filePath)
|| /\.stories\.(?:ts|tsx)$/.test(filePath)
|| /\.d\.ts$/.test(filePath)
}
export function getModuleName(filePath) {
const relativePath = filePath.slice(APP_COMPONENTS_PREFIX.length)
if (!relativePath)
return '(root)'
const segments = relativePath.split('/')
return segments.length === 1 ? '(root)' : segments[0]
}
export function isAnyComponentSourceFile(filePath) {
return filePath.startsWith(APP_COMPONENTS_PREFIX)
&& /\.(?:ts|tsx)$/.test(filePath)
&& !isTestLikePath(filePath)
}
export function isExcludedComponentSourceFile(filePath, excludedComponentCoverageFiles) {
return isAnyComponentSourceFile(filePath)
&& (
EXCLUDED_COMPONENT_MODULES.has(getModuleName(filePath))
|| excludedComponentCoverageFiles.has(filePath)
)
}
export function isTrackedComponentSourceFile(filePath, excludedComponentCoverageFiles) {
return isAnyComponentSourceFile(filePath)
&& !isExcludedComponentSourceFile(filePath, excludedComponentCoverageFiles)
}
export function isTrackedComponentTestFile(filePath) {
return filePath.startsWith(APP_COMPONENTS_PREFIX)
&& isTestLikePath(filePath)
&& !EXCLUDED_COMPONENT_MODULES.has(getModuleName(filePath))
}
export function isRelevantTestFile(filePath) {
return filePath.startsWith(SHARED_TEST_PREFIX)
|| isTrackedComponentTestFile(filePath)
}
export function isAnyWebTestFile(filePath) {
return filePath.startsWith('web/')
&& isTestLikePath(filePath)
}
export function getCoverageStats(entry) {
const lineHits = getLineHits(entry)
const statementHits = Object.values(entry.s ?? {})
const functionHits = Object.values(entry.f ?? {})
const branchHits = Object.values(entry.b ?? {}).flat()
return {
lines: {
covered: Object.values(lineHits).filter(count => count > 0).length,
total: Object.keys(lineHits).length,
},
statements: {
covered: statementHits.filter(count => count > 0).length,
total: statementHits.length,
},
functions: {
covered: functionHits.filter(count => count > 0).length,
total: functionHits.length,
},
branches: {
covered: branchHits.filter(count => count > 0).length,
total: branchHits.length,
},
}
}
export function sumCoverageStats(rows) {
const total = createEmptyCoverageStats()
for (const row of rows)
addCoverageStats(total, row)
return total
}
export function mergeCoverageStats(map, moduleName, stats) {
const existing = map.get(moduleName) ?? createEmptyCoverageStats()
addCoverageStats(existing, stats)
map.set(moduleName, existing)
}
export function percentage(covered, total) {
if (total === 0)
return 100
return (covered / total) * 100
}
export function formatPercent(metric) {
return `${percentage(metric.covered, metric.total).toFixed(2)}%`
}
function createEmptyCoverageStats() {
return {
lines: { covered: 0, total: 0 },
statements: { covered: 0, total: 0 },
functions: { covered: 0, total: 0 },
branches: { covered: 0, total: 0 },
}
}
function addCoverageStats(target, source) {
for (const metric of ['lines', 'statements', 'functions', 'branches']) {
target[metric].covered += source[metric].covered
target[metric].total += source[metric].total
}
}
function walkComponentSourceFiles(currentDir, onFile) {
if (!fs.existsSync(currentDir))
return
const entries = fs.readdirSync(currentDir, { withFileTypes: true })
for (const entry of entries) {
const entryPath = path.join(currentDir, entry.name)
if (entry.isDirectory()) {
if (entry.name === '__tests__' || entry.name === '__mocks__')
continue
walkComponentSourceFiles(entryPath, onFile)
continue
}
if (!/\.(?:ts|tsx)$/.test(entry.name) || isTestLikePath(entry.name))
continue
onFile(entryPath)
}
}

View File

@@ -1,10 +1,8 @@
import { createSecureServer } from 'node:http2'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
import { serve } from '@hono/node-server'
import { getCertificate } from '@vitejs/plugin-basic-ssl'
import { loadEnv } from 'vite'
import { createDevProxyApp, resolveDevProxyTargets, shouldUseHttpsForDevProxy } from '../plugins/dev-proxy/server'
import { createDevProxyApp, resolveDevProxyTargets } from '../plugins/dev-proxy/server'
const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..')
const mode = process.env.MODE || process.env.NODE_ENV || 'development'
@@ -13,33 +11,11 @@ const env = loadEnv(mode, projectRoot, '')
const host = env.HONO_PROXY_HOST || '127.0.0.1'
const port = Number(env.HONO_PROXY_PORT || 5001)
const app = createDevProxyApp(resolveDevProxyTargets(env))
const useHttps = shouldUseHttpsForDevProxy(env)
if (useHttps) {
const certificate = await getCertificate(
path.join(projectRoot, 'node_modules/.vite/basic-ssl'),
'localhost',
Array.from(new Set(['localhost', '127.0.0.1', host])),
)
serve({
fetch: app.fetch,
hostname: host,
port,
})
serve({
fetch: app.fetch,
hostname: host,
port,
createServer: createSecureServer,
serverOptions: {
allowHTTP1: true,
cert: certificate,
key: certificate,
},
})
}
else {
serve({
fetch: app.fetch,
hostname: host,
port,
})
}
console.log(`[dev-hono-proxy] listening on ${useHttps ? 'https' : 'http'}://${host}:${port}`)
console.log(`[dev-hono-proxy] listening on http://${host}:${port}`)

View File

@@ -1,165 +0,0 @@
import { execFileSync } from 'node:child_process'
import fs from 'node:fs'
import path from 'node:path'
import { COMPONENT_COVERAGE_EXCLUDE_LABEL } from './component-coverage-filters.mjs'
import {
collectTrackedComponentSourceFiles,
createComponentCoverageContext,
formatPercent,
getCoverageStats,
getModuleName,
loadTrackedCoverageEntries,
mergeCoverageStats,
percentage,
sumCoverageStats,
} from './components-coverage-common.mjs'
import {
COMPONENTS_GLOBAL_THRESHOLDS,
EXCLUDED_COMPONENT_MODULES,
getComponentModuleThreshold,
} from './components-coverage-thresholds.mjs'
const EXCLUDED_MODULES_LABEL = [...EXCLUDED_COMPONENT_MODULES].sort().join(', ')
const repoRoot = repoRootFromCwd()
const context = createComponentCoverageContext(repoRoot)
const coverageFinalPath = path.join(context.webRoot, 'coverage', 'coverage-final.json')
if (!fs.existsSync(coverageFinalPath)) {
console.error(`Coverage report not found at ${coverageFinalPath}`)
process.exit(1)
}
const coverage = JSON.parse(fs.readFileSync(coverageFinalPath, 'utf8'))
const trackedSourceFiles = collectTrackedComponentSourceFiles(context)
const coverageEntries = loadTrackedCoverageEntries(coverage, context)
const fileCoverageRows = []
const moduleCoverageMap = new Map()
for (const [file, entry] of coverageEntries.entries()) {
const stats = getCoverageStats(entry)
const moduleName = getModuleName(file)
fileCoverageRows.push({ file, moduleName, ...stats })
mergeCoverageStats(moduleCoverageMap, moduleName, stats)
}
const overallCoverage = sumCoverageStats(fileCoverageRows)
const overallTargetGaps = getTargetGaps(overallCoverage, COMPONENTS_GLOBAL_THRESHOLDS)
const moduleCoverageRows = [...moduleCoverageMap.entries()]
.map(([moduleName, stats]) => ({
moduleName,
stats,
targets: getComponentModuleThreshold(moduleName),
}))
.map(row => ({
...row,
targetGaps: row.targets ? getTargetGaps(row.stats, row.targets) : [],
}))
.sort((a, b) => {
const aWorst = Math.min(...a.targetGaps.map(gap => gap.delta), Number.POSITIVE_INFINITY)
const bWorst = Math.min(...b.targetGaps.map(gap => gap.delta), Number.POSITIVE_INFINITY)
return aWorst - bWorst || a.moduleName.localeCompare(b.moduleName)
})
appendSummary(buildSummary({
coverageEntriesCount: coverageEntries.size,
moduleCoverageRows,
overallCoverage,
overallTargetGaps,
trackedSourceFilesCount: trackedSourceFiles.length,
}))
function buildSummary({
coverageEntriesCount,
moduleCoverageRows,
overallCoverage,
overallTargetGaps,
trackedSourceFilesCount,
}) {
const lines = [
'### app/components Baseline Coverage',
'',
`Excluded modules: \`${EXCLUDED_MODULES_LABEL}\``,
`Excluded file kinds: \`${COMPONENT_COVERAGE_EXCLUDE_LABEL}\``,
'',
`Coverage entries: ${coverageEntriesCount}/${trackedSourceFilesCount} tracked source files`,
'',
'| Metric | Current | Target | Delta |',
'|---|---:|---:|---:|',
`| Lines | ${formatPercent(overallCoverage.lines)} | ${COMPONENTS_GLOBAL_THRESHOLDS.lines}% | ${formatDelta(overallCoverage.lines, COMPONENTS_GLOBAL_THRESHOLDS.lines)} |`,
`| Statements | ${formatPercent(overallCoverage.statements)} | ${COMPONENTS_GLOBAL_THRESHOLDS.statements}% | ${formatDelta(overallCoverage.statements, COMPONENTS_GLOBAL_THRESHOLDS.statements)} |`,
`| Functions | ${formatPercent(overallCoverage.functions)} | ${COMPONENTS_GLOBAL_THRESHOLDS.functions}% | ${formatDelta(overallCoverage.functions, COMPONENTS_GLOBAL_THRESHOLDS.functions)} |`,
`| Branches | ${formatPercent(overallCoverage.branches)} | ${COMPONENTS_GLOBAL_THRESHOLDS.branches}% | ${formatDelta(overallCoverage.branches, COMPONENTS_GLOBAL_THRESHOLDS.branches)} |`,
'',
]
if (coverageEntriesCount !== trackedSourceFilesCount) {
lines.push('Warning: coverage report did not include every tracked component source file. CI should set `VITEST_COVERAGE_SCOPE=app-components` before collecting coverage.')
lines.push('')
}
if (overallTargetGaps.length > 0) {
lines.push('Below baseline targets:')
for (const gap of overallTargetGaps)
lines.push(`- overall ${gap.metric}: ${gap.actual.toFixed(2)}% < ${gap.target}%`)
lines.push('')
}
lines.push('<details><summary>Module baseline coverage</summary>')
lines.push('')
lines.push('| Module | Lines | Statements | Functions | Branches | Targets | Status |')
lines.push('|---|---:|---:|---:|---:|---|---|')
for (const row of moduleCoverageRows) {
const targetsLabel = row.targets
? `L${row.targets.lines}/S${row.targets.statements}/F${row.targets.functions}/B${row.targets.branches}`
: 'n/a'
const status = row.targets
? (row.targetGaps.length > 0 ? 'below-target' : 'at-target')
: 'unconfigured'
lines.push(`| ${row.moduleName} | ${percentage(row.stats.lines.covered, row.stats.lines.total).toFixed(2)}% | ${percentage(row.stats.statements.covered, row.stats.statements.total).toFixed(2)}% | ${percentage(row.stats.functions.covered, row.stats.functions.total).toFixed(2)}% | ${percentage(row.stats.branches.covered, row.stats.branches.total).toFixed(2)}% | ${targetsLabel} | ${status} |`)
}
lines.push('</details>')
lines.push('')
lines.push('Report only: baseline targets no longer gate CI. The blocking rule is the pure diff coverage step.')
return lines
}
function getTargetGaps(stats, targets) {
const gaps = []
for (const metric of ['lines', 'statements', 'functions', 'branches']) {
const actual = percentage(stats[metric].covered, stats[metric].total)
const target = targets[metric]
const delta = actual - target
if (delta < 0) {
gaps.push({
actual,
delta,
metric,
target,
})
}
}
return gaps
}
function formatDelta(metric, target) {
const actual = percentage(metric.covered, metric.total)
const delta = actual - target
const sign = delta >= 0 ? '+' : ''
return `${sign}${delta.toFixed(2)}%`
}
function appendSummary(lines) {
const content = `${lines.join('\n')}\n`
if (process.env.GITHUB_STEP_SUMMARY)
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, content)
console.log(content)
}
function repoRootFromCwd() {
return execFileSync('git', ['rev-parse', '--show-toplevel'], {
cwd: process.cwd(),
encoding: 'utf8',
}).trim()
}

View File

@@ -1,168 +0,0 @@
import { execFileSync } from 'node:child_process'
import fs from 'node:fs'
import {
buildGitDiffRevisionArgs,
normalizeDiffRangeMode,
resolveGitDiffContext,
} from './check-components-diff-coverage-lib.mjs'
import {
createComponentCoverageContext,
isAnyWebTestFile,
isRelevantTestFile,
isTrackedComponentSourceFile,
} from './components-coverage-common.mjs'
const REQUESTED_DIFF_RANGE_MODE = normalizeDiffRangeMode(process.env.DIFF_RANGE_MODE)
const repoRoot = repoRootFromCwd()
const context = createComponentCoverageContext(repoRoot)
const baseSha = process.env.BASE_SHA?.trim()
const headSha = process.env.HEAD_SHA?.trim() || 'HEAD'
if (!baseSha || /^0+$/.test(baseSha)) {
appendSummary([
'### app/components Test Touch',
'',
'Skipped test-touch report because `BASE_SHA` was not available.',
])
process.exit(0)
}
const diffContext = resolveGitDiffContext({
base: baseSha,
head: headSha,
mode: REQUESTED_DIFF_RANGE_MODE,
execGit,
})
const changedFiles = getChangedFiles(diffContext)
const changedSourceFiles = changedFiles.filter(filePath => isTrackedComponentSourceFile(filePath, context.excludedComponentCoverageFiles))
if (changedSourceFiles.length === 0) {
appendSummary([
'### app/components Test Touch',
'',
...buildDiffContextSummary(diffContext),
'',
'No tracked source changes under `web/app/components/`. Test-touch report skipped.',
])
process.exit(0)
}
const changedRelevantTestFiles = changedFiles.filter(isRelevantTestFile)
const changedOtherWebTestFiles = changedFiles.filter(filePath => isAnyWebTestFile(filePath) && !isRelevantTestFile(filePath))
const totalChangedWebTests = [...new Set([...changedRelevantTestFiles, ...changedOtherWebTestFiles])]
appendSummary(buildSummary({
changedOtherWebTestFiles,
changedRelevantTestFiles,
diffContext,
changedSourceFiles,
totalChangedWebTests,
}))
function buildSummary({
changedOtherWebTestFiles,
changedRelevantTestFiles,
diffContext,
changedSourceFiles,
totalChangedWebTests,
}) {
const lines = [
'### app/components Test Touch',
'',
...buildDiffContextSummary(diffContext),
'',
`Tracked source files changed: ${changedSourceFiles.length}`,
`Component-local or shared integration tests changed: ${changedRelevantTestFiles.length}`,
`Other web tests changed: ${changedOtherWebTestFiles.length}`,
`Total changed web tests: ${totalChangedWebTests.length}`,
'',
]
if (totalChangedWebTests.length === 0) {
lines.push('Warning: no frontend test files changed alongside tracked component source changes.')
lines.push('')
}
if (changedRelevantTestFiles.length > 0) {
lines.push('<details><summary>Changed component-local or shared tests</summary>')
lines.push('')
for (const filePath of changedRelevantTestFiles.slice(0, 40))
lines.push(`- ${filePath.replace('web/', '')}`)
if (changedRelevantTestFiles.length > 40)
lines.push(`- ... ${changedRelevantTestFiles.length - 40} more`)
lines.push('</details>')
lines.push('')
}
if (changedOtherWebTestFiles.length > 0) {
lines.push('<details><summary>Changed other web tests</summary>')
lines.push('')
for (const filePath of changedOtherWebTestFiles.slice(0, 40))
lines.push(`- ${filePath.replace('web/', '')}`)
if (changedOtherWebTestFiles.length > 40)
lines.push(`- ... ${changedOtherWebTestFiles.length - 40} more`)
lines.push('</details>')
lines.push('')
}
lines.push('Report only: test-touch is now advisory and no longer blocks the diff coverage gate.')
return lines
}
function buildDiffContextSummary(diffContext) {
const lines = [
`Compared \`${diffContext.base.slice(0, 12)}\` -> \`${diffContext.head.slice(0, 12)}\``,
]
if (diffContext.useCombinedMergeDiff) {
lines.push(`Requested diff range mode: \`${diffContext.requestedMode}\``)
lines.push(`Effective diff strategy: \`combined-merge\` (${diffContext.reason})`)
}
else if (diffContext.reason) {
lines.push(`Requested diff range mode: \`${diffContext.requestedMode}\``)
lines.push(`Effective diff range mode: \`${diffContext.mode}\` (${diffContext.reason})`)
}
else {
lines.push(`Diff range mode: \`${diffContext.mode}\``)
}
return lines
}
function getChangedFiles(diffContext) {
if (diffContext.useCombinedMergeDiff) {
const output = execGit(['diff-tree', '--cc', '--no-commit-id', '--name-only', '-r', diffContext.head, '--', 'web'])
return output
.split('\n')
.map(line => line.trim())
.filter(Boolean)
}
const output = execGit(['diff', '--name-only', '--diff-filter=ACMR', ...buildGitDiffRevisionArgs(diffContext.base, diffContext.head, diffContext.mode), '--', 'web'])
return output
.split('\n')
.map(line => line.trim())
.filter(Boolean)
}
function appendSummary(lines) {
const content = `${lines.join('\n')}\n`
if (process.env.GITHUB_STEP_SUMMARY)
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, content)
console.log(content)
}
function execGit(args) {
return execFileSync('git', args, {
cwd: repoRoot,
encoding: 'utf8',
})
}
function repoRootFromCwd() {
return execFileSync('git', ['rev-parse', '--show-toplevel'], {
cwd: process.cwd(),
encoding: 'utf8',
}).trim()
}

View File

@@ -1,12 +1,9 @@
import path from 'node:path'
import { fileURLToPath } from 'node:url'
import basicSsl from '@vitejs/plugin-basic-ssl'
import react from '@vitejs/plugin-react'
import vinext from 'vinext'
import { loadEnv } from 'vite'
import Inspect from 'vite-plugin-inspect'
import { defineConfig } from 'vite-plus'
import { shouldUseHttpsForDevProxy } from './plugins/dev-proxy/protocol'
import { createCodeInspectorPlugin, createForceInspectorClientInjectionPlugin } from './plugins/vite/code-inspector'
import { customI18nHmrPlugin } from './plugins/vite/custom-i18n-hmr'
import { nextStaticImageTestPlugin } from './plugins/vite/next-static-image-test'
@@ -24,8 +21,6 @@ export default defineConfig(({ mode }) => {
const isTest = mode === 'test'
const isStorybook = process.env.STORYBOOK === 'true'
|| process.argv.some(arg => arg.toLowerCase().includes('storybook'))
const env = loadEnv(mode, projectRoot, '')
const useHttpsForDevServer = shouldUseHttpsForDevProxy(env)
const isAppComponentsCoverage = coverageScope === 'app-components'
const excludedComponentCoverageFiles = isAppComponentsCoverage
? collectComponentCoverageExcludedFiles(path.join(projectRoot, 'app/components'), { pathPrefix: 'app/components' })
@@ -62,7 +57,6 @@ export default defineConfig(({ mode }) => {
react(),
vinext({ react: false }),
customI18nHmrPlugin({ injectTarget: browserInitializerInjectTarget }),
...(useHttpsForDevServer ? [basicSsl()] : []),
// reactGrabOpenFilePlugin({
// injectTarget: browserInitializerInjectTarget,
// projectRoot,