Compare commits

..

3 Commits

Author SHA1 Message Date
yyh
59912d10fc refactor(web): remove code generator React.FC 2026-08-09 17:46:48 +08:00
yyh
b975ed698e refactor(web): use css icon for code generator 2026-08-09 17:45:32 +08:00
yyh
03ca1730c7 fix(web): name code generator action 2026-08-09 17:44:35 +08:00
9243 changed files with 266417 additions and 1206516 deletions

View File

@@ -41,7 +41,6 @@
# core/policy/publish_policy.py
from controllers.console.app import request_context
def can_publish() -> bool:
return request_context.current_user.is_admin
```
@@ -51,7 +50,6 @@
def can_publish(role: str) -> bool:
return role == "admin"
# service layer adapts web/user context to domain input
allowed = can_publish(role=current_user.role)
```
@@ -68,7 +66,6 @@
# api/libs/conversation_filter.py
from services.conversation_service import ConversationService
def should_archive_conversation(conversation, tenant_id: str) -> bool:
# Domain policy and service dependency are leaking into libs.
service = ConversationService()
@@ -82,11 +79,9 @@
def older_than_days(idle_days: int, threshold_days: int) -> bool:
return idle_days > threshold_days
# services/conversation_service.py (business logic stays in service/core)
from libs.datetime_utils import older_than_days
def should_archive_conversation(conversation, tenant_id: str) -> bool:
threshold_days = 90 if has_paid_plan(tenant_id) else 30
return older_than_days(conversation.idle_days, threshold_days)

View File

@@ -50,7 +50,6 @@
```python
from sqlalchemy.orm import Mapped
class Dataset(TypeBase):
__tablename__ = "datasets"
id: Mapped[str] = mapped_column(StringUUID, primary_key=True)
@@ -60,7 +59,6 @@
```python
from sqlalchemy.orm import Mapped
class Dataset(TypeBase):
__tablename__ = "datasets"
id: Mapped[str] = mapped_column(StringUUID, primary_key=True)
@@ -103,7 +101,6 @@
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped
class ToolConfig(TypeBase):
__tablename__ = "tool_configs"
config: Mapped[dict] = mapped_column(JSONB, nullable=False)
@@ -114,7 +111,6 @@
from models.types import AdjustedJSON
class ToolConfig(TypeBase):
__tablename__ = "tool_configs"
config: Mapped[dict] = mapped_column(AdjustedJSON(), nullable=False)
@@ -150,5 +146,7 @@
default_expr = sa.text("'database'::character varying") if _is_pg(conn) else sa.text("'database'")
with op.batch_alter_table("dataset_keyword_tables") as batch_op:
batch_op.add_column(sa.Column("data_source_type", sa.String(255), server_default=default_expr, nullable=False))
batch_op.add_column(
sa.Column("data_source_type", sa.String(255), server_default=default_expr, nullable=False)
)
```

View File

@@ -18,7 +18,9 @@
# Existing repository is ignored and service uses ad-hoc table queries.
class AppService:
def archive_app(self, app_id: str, tenant_id: str) -> None:
app = self.session.execute(select(App).where(App.id == app_id, App.tenant_id == tenant_id)).scalar_one()
app = self.session.execute(
select(App).where(App.id == app_id, App.tenant_id == tenant_id)
).scalar_one()
app.archived = True
self.session.commit()
```
@@ -31,7 +33,6 @@
app.archived = True
self.app_repo.save(app)
# If the query is missing, extend the existing abstraction.
active_apps = self.app_repo.list_active_for_tenant(tenant_id=tenant_id)
```
@@ -49,10 +50,9 @@
class ConversationRepository(Protocol):
def list_recent_for_app(self, app_id: str, tenant_id: str, limit: int) -> list[Conversation]: ...
class SqlAlchemyConversationRepository:
def list_recent_for_app(self, app_id: str, tenant_id: str, limit: int) -> list[Conversation]: ...
def list_recent_for_app(self, app_id: str, tenant_id: str, limit: int) -> list[Conversation]:
...
class ConversationService:
def __init__(self, conversation_repo: ConversationRepository):

View File

@@ -126,7 +126,9 @@
# 3) Pessimistic lock with SELECT ... FOR UPDATE (high contention)
run = session.execute(
select(WorkflowRun).where(WorkflowRun.id == run_id, WorkflowRun.tenant_id == tenant_id).with_for_update()
select(WorkflowRun)
.where(WorkflowRun.id == run_id, WorkflowRun.tenant_id == tenant_id)
.with_for_update()
).scalar_one()
run.status = "cancelled"
session.commit()

View File

@@ -7,14 +7,12 @@ description: Use when writing, changing, or reviewing Cucumber and Playwright te
`e2e/AGENTS.md` owns the suite architecture, lifecycle, commands, tags, generated-client boundaries, fixtures, and cleanup contracts. Read the nearest feature-scoped `AGENTS.md` when one exists. This skill adds no parallel package policy.
Cucumber owns scenario and hook execution and reporting. Playwright provides browser automation, contexts, pages, locators, actions, assertions, request, and tracing APIs; protocol exceptions remain with the package-defined owner. Their timeout domains remain separate, and importing from `@playwright/test` does not make Playwright Test runner configuration, fixtures, or retries applicable to this suite.
## Topic Routing
Read only the bundled reference required by the change:
- Locator, assertion, isolation, or waiting decisions: [`references/playwright-best-practices.md`][playwright]
- Scenario wording, step granularity, expressions, World state, hook visibility, or tag design: [`references/cucumber-best-practices.md`][cucumber]
- Scenario wording, step granularity, expressions, or tag design: [`references/cucumber-best-practices.md`][cucumber]
Check current official Playwright or Cucumber documentation before introducing a framework pattern that local code and references do not already establish.
@@ -25,7 +23,7 @@ Check current official Playwright or Cucumber documentation before introducing a
3. Read the target scenario, matching step definitions, and lifecycle files only when session or shared state matters.
4. Reuse an existing step when wording and behavior match; add one coherent scenario or step when they do not.
5. Keep browser actions and assertions at the public user boundary; keep setup, seed, polling, and cleanup at their package-defined owners.
6. For changes, run the narrowest tagged scenario and package checks documented in `e2e/AGENTS.md`; broaden only for shared hooks, tags, or support changes.
6. Run the narrowest tagged scenario and package checks documented in `e2e/AGENTS.md`; broaden only for shared hooks, tags, or support changes.
For review requests, lead with reproducible correctness failures, flake sources, or demonstrated architecture drift. Report the behavior verified and any external-runtime, browser, or environment gap.

View File

@@ -4,18 +4,15 @@ Use this reference when writing or reviewing Gherkin scenarios, step definitions
Official sources:
- https://cucumber.io/docs/bdd/better-gherkin
- https://cucumber.io/docs/gherkin/reference
- https://cucumber.io/docs/gherkin/step-organization
- https://cucumber.io/docs/cucumber/state
- https://cucumber.io/docs/guides/10-minute-tutorial
- https://cucumber.io/docs/cucumber/step-definitions
- https://cucumber.io/docs/cucumber/cucumber-expressions
- https://github.com/cucumber/cucumber-js/blob/main/docs/support_files/world.md
## What Matters Most
### 1. Treat scenarios as executable specifications
Cucumber scenarios should usually describe behavior declaratively, not reproduce an interaction script.
Cucumber scenarios should describe examples of behavior, not test implementation recipes.
Apply it like this:
@@ -23,15 +20,11 @@ Apply it like this:
- avoid UI-internal wording such as selector details, DOM structure, or component names
- keep language concrete enough that the scenario reads like living documentation
Procedural detail is appropriate when the interaction mechanism is the behavior under test, such as keyboard navigation, focus movement, or a required multi-actor sequence.
### 2. Keep scenarios focused
A scenario should prove one business rule or coherent outcome. Cucumber's three-to-five-step recommendation is a review heuristic, not a hard limit: when a scenario is longer, check for multiple outcomes, incidental setup, or UI procedure that can move behind a domain step.
A scenario should usually prove one workflow or business outcome. If a scenario wanders across several unrelated behaviors, split it.
Use `Given` for initial context, `When` for the event, and `Then` for the expected outcome. Restarting those phases is a prompt to review the narrative, not an automatic failure. Avoid hidden dependencies on another scenario's side effects.
Use `Rule` when several examples illustrate one named business rule. Use a short `Background` only for shared context the reader needs to understand; do not hide fixture preparation, runtime readiness, or long setup there.
Keep each scenario centered on one coherent outcome. Avoid hidden dependencies on another scenario's side effects, and keep unavoidable setup outside the behavior narrative unless the precondition matters to the specification.
### 3. Reuse steps, but only when behavior really matches
@@ -52,8 +45,6 @@ Write a new step when:
Do not optimize for a low step count by making vague steps. Optimize for a small set of truthful, domain-owned steps.
Step definitions share one global matching namespace regardless of their directory. Organize them by domain capability and avoid feature-coupled glue or expressions broad enough to overlap unrelated behavior.
### 4. Prefer Cucumber Expressions
Use Cucumber Expressions for parameters unless regex is clearly necessary.
@@ -71,24 +62,18 @@ Use regex for a bounded natural-language alternative only when it keeps Gherkin
### 5. Keep step definitions thin and meaningful
Step definitions are glue between Gherkin and automation, not a second abstraction language. A step should express one coherent domain action or outcome; it may call several implementation helpers when that keeps UI mechanics out of the specification.
Step definitions are glue between Gherkin and automation, not a second abstraction language.
Cucumber creates a new World for each scenario. Keep scenario state there rather than in module globals. Cucumber.js supports a `world` proxy for arrow functions, but this suite consistently uses typed `async function (this: DifyWorld, ...)` so ownership and TypeScript discovery remain explicit.
Hooks are invisible to feature readers. Keep them for low-level browser lifecycle, cleanup, and diagnostics; express business-relevant context in Gherkin.
Keep each step to one user-visible action or assertion. In JavaScript and TypeScript, use `async function` when the step reads Cucumber World state because Cucumber binds `this`; do not leak state across scenarios through module globals.
### 6. Use tags intentionally
Tags may communicate stable capability grouping, selection, fixture dependency, or conditional-hook intent. A tag changes runtime behavior only when a runner, seed profile, or hook owns that meaning.
Tags should communicate selection or execution intent, not become ad hoc metadata. A tag does not change runtime behavior unless configuration or hooks implement it.
## Review Questions
- Does the scenario read like a real example of product behavior?
- Does it prove one outcome, or combine several independent phases?
- Are the steps behavior-oriented instead of implementation-oriented?
- Is procedural wording necessary to the behavior being tested?
- Is a reused step still truthful in this feature?
- Could a new expression overlap an existing step in the global namespace?
- Would `Rule` clarify the business rule, or would `Background` hide important setup?
- Is a new tag documenting real behavior, or inventing semantics that the suite does not implement?
- Would a new reader understand the outcome without opening the step-definition file?

View File

@@ -6,12 +6,8 @@ Official sources:
- https://playwright.dev/docs/best-practices
- https://playwright.dev/docs/locators
- https://playwright.dev/docs/actionability
- https://playwright.dev/docs/test-assertions
- https://playwright.dev/docs/test-timeouts
- https://playwright.dev/docs/browser-contexts
- https://playwright.dev/docs/events
- https://playwright.dev/docs/trace-viewer
## What Matters Most
@@ -25,22 +21,28 @@ Apply it like this:
- keep scenario state in the runner's scenario-owned context rather than module globals
- model special authentication or session setup through explicit per-scenario fixtures rather than shared mutable state
### 2. Select locators by user contract
### 2. Prefer user-facing locators
Prefer built-in locators that match how the target is exposed to users:
Playwright recommends built-in locators that reflect what users perceive on the page.
- interactive controls: role and accessible name
- form controls: associated label; use placeholder when it is the relevant stable contract, especially when no label exists
- non-interactive content: visible text or the relevant text alternative
- elements without a meaningful user-facing contract: an intentional test id
Preferred order:
This is a semantic choice, not a fixed ranking. Do not add incorrect roles or accessible names solely to satisfy a locator. When the product element should have user-facing semantics, fix that contract instead.
1. `getByRole`
2. `getByLabel`
3. `getByPlaceholder`
4. `getByText`
5. `getByTestId` when an explicit test contract is the most stable option
Avoid raw CSS/XPath selectors unless no stable user-facing contract exists and adding one is not practical.
Locators are strict for single-element actions. Scope to a stable region or use `filter({ has, hasText })` to make the intended target unique. Treat `.first()`, `.last()`, or `.nth()` as a review signal: the positional choice should be intentional and stable, not merely silence ambiguity.
Also remember:
### 3. Use web-first assertions with the right timeout owner
- repeated content usually needs scoping to a stable container
- exact text matching is often too brittle when role/name or label already exists
- `getByTestId` is acceptable when semantics are weak but the contract is intentional
- when a real UI region, card, status, or icon lacks an accessible name, prefer adding that semantic contract in product code before falling back to `getByTestId`
### 3. Use web-first assertions
Playwright assertions auto-wait and retry. Prefer them over manual state inspection.
@@ -62,8 +64,6 @@ If a condition genuinely needs custom retry logic, use Playwright's polling/asse
Use `expect.poll` for non-DOM truth such as API state, backend eventual consistency, generated resources, or captured browser events. For DOM state, use locator assertions so Playwright can apply actionability and web-first retry semantics.
Cucumber step and hook timeouts, Playwright locator/action timeouts, and Playwright assertion timeouts are separate budgets. `browserContext.setDefaultTimeout()` does not change the default five-second assertion timeout. Put an explicit longer assertion timeout only on the readiness owner that needs it; do not raise an outer timeout to mask a shorter inner failure.
### 4. Let actions wait for actionability
Locator actions already wait for the element to be actionable. Do not preface every click/fill with extra timing logic unless the action needs a specific visible/ready assertion for clarity.
@@ -77,20 +77,15 @@ Bad pattern:
- stack arbitrary waits before every action
- wait on unstable implementation details instead of the visible state the user cares about
- use `force: true` to bypass a real hit-target, overlay, or disabled-state failure
When synchronizing on a one-shot popup, download, request, or response, create the wait before triggering the action, then await the event. A scenario-owned listener may instead capture events for a later assertion. Do not use `networkidle` as an application-readiness assertion; wait for the user-visible state or owned backend contract.
### 5. Match debugging to the active harness
Playwright supports traces, screenshots, page snapshots, and browser logs. Configure artifact capture in Cucumber hooks instead of adding parallel diagnostics to individual scenarios. If tracing is introduced, use `browserContext.tracing`; Playwright Test options such as `trace: 'on-first-retry'`, projects, workers, fixtures, reporters, and retries do not configure this harness.
Playwright supports traces, screenshots, page snapshots, and browser logs. Configure artifact capture at the runner boundary instead of adding parallel diagnostics to individual scenarios.
## Review Questions
- Would this locator survive DOM refactors that do not change user-visible behavior?
- Is a positional locator expressing product order, or hiding an ambiguous match?
- Is this assertion using Playwright's retrying semantics?
- Does an explicit timeout belong to the condition that is actually slow?
- Was an event wait registered before the action that emits it?
- Is any explicit wait masking a real readiness problem?
- Does this code preserve per-scenario isolation?
- Is a new abstraction really needed, or does it bypass the runner's scenario-owned context and lifecycle?

View File

@@ -1,19 +1,18 @@
---
name: frontend-code-review
description: Use only when the user explicitly requests a bug, regression, correctness, security, accessibility, performance, or test-quality review of frontend code under `web/` or `packages/dify-ui/`. Supports pending-change, file-focused, and pasted-diff reviews. Do not use for architecture-oriented evaluation of component ownership, state graphs, props, boundaries, or refactoring design unless concrete defects are also requested; do not use for implementation-only requests, diagnosis without review intent, or backend-only code.
description: Use only when the user explicitly requests a review or audit of frontend code under `web/` or `packages/dify-ui/`. Supports pending-change, file-focused, and pasted-diff reviews. Do not use for implementation-only requests, diagnosis without review intent, or backend-only code.
---
# Frontend Code Review
Review the requested scope for concrete defects and violations of explicit project contracts. This skill owns review decisions; its references route to canonical rules without activating another skill's implementation workflow.
Review the requested scope for concrete, reproducible regressions. This skill owns the review phase and routes directly to its bundled rule packs. For a combined review-and-fix request, establish findings before applying implementation or testing guidance.
## Evidence First
1. Establish the review scope from the requested files or current diff.
2. Read the changed lines, their behavior owner, and the nearest scoped `AGENTS.md`.
3. For a named component or a diff that changes component ownership, props, or state, read the component-architecture rule pack and complete its Root-To-Leaf State And Props Audit before drawing conclusions.
4. Trace other public consumers, generated contracts, primitive APIs, or runtime configuration only when they decide correctness.
5. Report findings tied to an observable failure, violated contract, security boundary, or demonstrated maintenance risk. Explicit team conventions are contracts: establish their scope and exceptions, and do not invent user impact to justify a convention finding.
3. Trace public consumers, generated contracts, primitive APIs, or runtime configuration only when they decide correctness.
4. Report only findings tied to an observable failure, violated contract, security boundary, or demonstrated maintenance risk.
## Rule Routing
@@ -28,16 +27,16 @@ Read only the packs matched by the diff:
- Stable Dify runtime invariants in the named paths: [`references/dify-invariants.md`][dify-invariants]
- General TypeScript or styling quality not owned above: [`references/code-quality.md`][code-quality]
Read `packages/dify-ui/README.md`, `packages/dify-ui/AGENTS.md`, `packages/dify-ui/docs/overlays.md`, or `web/docs/test.md` only when the reviewed code falls under that contract. Check current official documentation when local code and bundled references do not settle a framework, browser, or accessibility behavior.
Read `packages/dify-ui/README.md`, `packages/dify-ui/AGENTS.md`, `web/docs/overlay.md`, or `web/docs/test.md` only when the reviewed code falls under that contract. Check current official documentation when local code and bundled references do not settle a framework, browser, or accessibility behavior.
## Severity And Output
- **P0**: security or privacy leak, data loss, production crash, or inaccessible critical workflow.
- **P1**: user-visible regression, invalid API or authorization contract, hydration failure, or broken primary interaction.
- **P2**: concrete maintainability, performance, test, or accessibility defect, or a material violation of an explicit project contract.
- **P2**: concrete maintainability, performance, test, or accessibility defect likely to cause incorrect behavior.
- **P3**: minor actionable cleanup; omit unless the user requested a thorough audit.
Lead with findings ordered by severity. Include a tight file and line reference, the observed failure or applicable project rule, and a concrete fix direction. Explain the rule's applicability for convention findings; describe downstream consequences only when supported by evidence. When no findings remain, say so briefly and state any material verification gap. Do not add praise sections, speculative risks, or an unsolicited offer to implement fixes.
Lead with findings ordered by severity. Include a tight file and line reference, the failing contract or reproduction path, impact, and a concrete fix direction. If there are no findings, say `No issues found.` and state any material verification gap. Do not add praise sections, speculative risks, or an unsolicited offer to implement fixes.
[accessibility]: references/accessibility-ui.md
[code-quality]: references/code-quality.md

View File

@@ -2,51 +2,32 @@
Accessibility findings are first-class review findings. Treat broken keyboard access, missing accessible names, focus loss, and unreachable popup content as correctness bugs, not polish.
## Review Evidence
Before finalizing UI or accessibility findings, fetch the latest Web Interface Guidelines as a required baseline:
Use the sources relevant to the changed contract:
```text
https://raw.githubusercontent.com/vercel-labs/web-interface-guidelines/main/command.md
```
Do not treat that document as the complete accessibility rule set. Combine it with:
- `packages/dify-ui/README.md`, `packages/dify-ui/AGENTS.md`, and the relevant primitive implementation when code uses `@langgenius/dify-ui/*`.
- Base UI docs and local `.d.ts` contracts when primitive semantics, focus target, labels, or popup reachability are unclear.
- MDN or relevant WAI-ARIA/browser standards when behavior, compatibility, or deprecation status matters.
- The current feature's product semantics, because an accessible primitive can still be used in an inaccessible workflow.
Consult current official documentation or standards when these sources leave a behavior unresolved. The [Web Interface Guidelines] are an optional broader UI reference, not a required fetch for each review.
## Semantic HTML
Flag:
- Clickable `div` or `span` used for actions.
- Router navigation implemented with button or `onClick` when a `Link` / `<a>` is the real semantic element.
- Icon-only controls without an accessible name; follow the naming rules below and the Dify UI `IconButton` contract.
- Icon-only buttons without `aria-label` or `aria-labelledby`.
- Decorative icons missing `aria-hidden="true"`.
- Images without `alt`; use `alt=""` only when truly decorative.
- Heading levels that skip hierarchy in page-level content.
Prefer semantic HTML before ARIA.
## Page Landmarks
When reviewing Web page layouts, navigation, or landmark changes, read [Page landmarks].
Inspect the composed page with its parent layouts for missing or duplicate main regions,
inappropriate nesting or roles, and ambiguous or broken landmark names. Check affected
unit and E2E locators when semantic elements change. Web owns these composition rules;
use the existing naming contract for label-source decisions.
## Accessible Names And Descriptions
Read [Accessible names and descriptions] when a change affects labels, ARIA naming, help/error relationships, or hidden text. That document owns the shared implementation and review contract.
Flag violations supported by the final rendered behavior:
- Missing or insufficient names, redundant name overrides, or naming attributes prohibited by the element's role.
- Overrides that omit visible label wording or suppress necessary descendant information.
- Broken or stale label/description references, including relationships lost when responsive content or overlays unmount.
- Descriptions used instead of names, repeated help text, or essential structured content available only as a flattened description.
Inspect the computed name and description in the relevant state. Matching an `aria-label` string to nearby text alone does not prove redundancy.
## Keyboard And Focus
Flag:
@@ -67,6 +48,7 @@ Flag:
- Missing stable `name` on form fields that submit or validate.
- Incorrect input `type`, `inputMode`, `autoComplete`, or `spellCheck` for email, token, URL, number, search, code, or username fields.
- Labels that are not clickable.
- Submit buttons disabled before a request starts, preventing normal submit behavior.
- Non-submit buttons inside forms missing `type="button"`.
- Errors not associated with fields or not reachable by screen readers.
- Error recovery that does not focus or expose the first invalid field.
@@ -74,20 +56,16 @@ Flag:
- Placeholder text used as the only label.
- Password managers accidentally triggered on non-auth fields because autocomplete is missing or wrong.
Prefer visible labels and associate them through the appropriate field primitive, a native `label`, or `aria-labelledby`. Do not duplicate an existing label with hidden text or `aria-label`; follow [Accessible names and descriptions] when no suitable visible label exists.
Prefer visible labels. If visible surrounding text already labels the control, use a visually hidden label or a precise `aria-label`.
## Disabled, Loading, And Async States
Flag:
- Loading controls whose accessible name disappears, or whose user-relevant progress has no
feature-owned status path. Follow the Dify UI Button contract for focused loading buttons; do not
add `aria-busy` to a button as a generic pending-state substitute.
- The same pending state passed to both Dify UI Button `loading` and `disabled`, which duplicates
state ownership and obscures whether `disabled` expresses independent unavailability.
- Loading state without `aria-busy`, `role="status"`, or another accessible update path when it changes user interaction.
- Spinner or decorative loading icon exposed to screen readers.
- Disabled controls that hide the reason users cannot proceed.
- Controls marked `aria-disabled` that still activate through supported pointer or keyboard input. Check the primitive's handling before requesting manual event guards.
- `aria-disabled` used without manually blocking click, Space, and Enter.
- Toasts, inline validation, or async status changes that are not announced when users need the update to continue.
- Icon-only loading/error affordances without text or accessible status where the state matters.
@@ -112,7 +90,7 @@ Use Popover for explanatory content, rich help, and infotips. Use Tooltip only a
Flag:
- Text in flex/grid children without `min-w-0` when it can overflow.
- Long names, labels, or user content that overflow, obscure adjacent controls, or become unreadable in supported layouts.
- Names, labels, file names, model names, workspace names, or user content lacking `truncate`, `line-clamp`, or `break-words`.
- Right-side icons, badges, checks, or actions that shrink before the text area.
- Empty arrays or empty strings rendering broken layout instead of an empty state.
- Button, tab, badge, chip, menu item, or card text that can overlap sibling controls at common viewport widths.
@@ -129,7 +107,3 @@ Flag:
- Images without dimensions.
- Loading copy using `...` instead of `…`.
- Hardcoded dates, times, numbers, or currency formats instead of `Intl.*`.
[Accessible names and descriptions]: ../../../../packages/dify-ui/docs/accessible-names-and-descriptions.md
[Page landmarks]: ../../../../web/docs/landmarks.md
[Web Interface Guidelines]: https://raw.githubusercontent.com/vercel-labs/web-interface-guidelines/main/command.md

View File

@@ -33,7 +33,7 @@ Flag:
- Generic color utilities where Dify semantic tokens exist.
- Hardcoded magic class values for colors, spacing, radius, shadow, z-index, or typography when Dify tokens, component variants, or documented radius mappings exist.
- `!` important modifiers or important CSS overrides without a narrow, documented reason.
- Manual class-list assembly through string concatenation, template strings, array `.join(' ')`, or a custom conditional combiner instead of `cn(...)`. Conditional values passed to `cn(...)` remain valid.
- Manual string concatenation, template strings, array `.join(' ')`, or custom ternaries for conditional or multi-line classes.
- JS conditional class branches for primitive visual states already exposed by Dify UI/Base UI `data-*` selectors.
- Incoming `className` placed before default classes in `cn(...)`, preventing call-site overrides.
- Arbitrary z-index or one-off layering fixes on overlays.

View File

@@ -1,44 +1,88 @@
# Component Architecture Review
# Component Architecture Rules
Use the canonical reference for the changed concern. These links share rules; they do not activate the implementation skill or its workflow.
Use these rules for React component structure, ownership, state, props, effects, and module organization.
| Concern | Canonical rules |
| --- | --- |
| Vertical modules, public entrypoints, data/handler placement, wrappers, Props, and types | [Ownership] |
| Local/Jotai state, form drafts, route identity, URL state, and persistence | [State] |
| Effects, navigation, memoization, and subscriptions | [Runtime] |
| Hotkeys, focus, and secondary surfaces | [Interactions] |
## Ownership
## Root-To-Leaf State And Props Audit
Flag:
For a file-focused component review, use the named component as the root. For a pending diff, use each changed page, feature entrypoint, or nearest behavior owner whose state or props contract changed.
- State, query, mutation, or handlers hoisted above the lowest component that actually uses them.
- Parent components owning row/item actions that do not coordinate a workflow.
- Prop drilling through multiple pass-through layers.
- A page/tab-level section component becoming the data owner without needing a shared snapshot or shared loading/error/empty UI.
- Feature code promoted to shared only because it appears once or might be reused later.
1. Build the locally owned rendered component tree from each root to every leaf. Follow imported feature components, conditional branches, lists, portals, dialogs, drawers, and popovers. Stop at third-party or Dify UI primitives and explicit stable public feature boundaries, but record the props passed across that final edge.
2. At every component, inventory state sources: React state/reducers, form state, URL state, context/store/Jotai reads and writes, queries, mutations, refs that hold workflow state, and custom-hook results.
3. For every state value, derived fact, and handler, record its consuming branches and required lifetime, including persistence across descendant unmounts. Assess its placement against [Ownership] and [State], including their coordination and persistence exceptions.
4. Inspect every props edge. Mark values that are read by the child, forwarded unchanged, renamed, recomputed, mirrored into local state, or paired with lifecycle fields such as `data/pending/error/retry` and `open/onOpenChange`.
5. Trace what each forwarding parent actually owns before reporting an unnecessary intermediary. Apply [Ownership] to distinguish required coordination or persistence from merely redistributing workflow fields, and [State] to determine whether a feature graph is warranted.
6. Remove unused or redundant props at their source. Do not accept replacing many props with a props bag, context, or hook result unless that abstraction becomes the real owner and gives consumers a narrower contract.
Accept repeated TanStack Query calls in siblings when each component independently consumes the data. Cache deduplication is not a reason to hoist by itself.
Review the complete paths before concluding that a state or prop is necessary. When several edges share one ownership defect, report one finding at the highest incorrect owner and include a representative path such as `Root -> Section -> Leaf`; do not emit a duplicate finding for every child.
## Component Boundaries
## Apply Rules In Their Actual Scope
Flag:
Explicit team conventions are reviewable contracts, including module organization and public API boundaries. Check the documented exception before reporting a violation. Do not infer an exception solely because the code appears to work, or invent a user-facing failure for a convention finding.
- React component files over 300 lines when the file mixes multiple responsibilities that can be split into focused colocated components, hooks, or utilities.
- Shallow wrappers that only rename props or hide the real primitive.
- Extra DOM wrappers that do not provide layout, semantics, accessibility, state ownership, or library integration.
- Dialog/dropdown/popover hidden surfaces that obscure the parent flow when they should be extracted into a small local component.
- Business forms, menu bodies, or one-off helpers moved away from their owner without reuse or semantic value.
- For owner placement, trace the consumers and required lifetime. Establish whether the parent coordinates a snapshot, submission, navigation, shared UI, or persistence before asking to move state or handlers.
- For component boundaries, identify the ownership or encapsulation the proposed extraction would improve; file length alone establishes neither.
- For props and types, check the domain contract and public API. Do not report private props typing style alone; declaration/export syntax matters only for a documented package rule or concrete type, export, or framework defect.
- For state and Effects, trace the source of truth, external synchronization target, and mount/reset boundary. Controlledness alone does not prove that a draft is lifted or persisted; follow the form and overlay contracts linked by [State].
- For navigation, distinguish ordinary links from mutation success, guarded redirects, command flows, and submission side effects.
Prefer colocated components split by actual data and state needs.
## Preserve Existing Product Contracts
## Bad Component Design Patterns
During refactors, trace the interaction being moved through its real consumer. Navigation, sidebar, dropdown, webapp-list, and app-switching changes must preserve expansion controls, hover persistence, pin/delete actions, routing, keyboard/focus handling, and open-state ownership where present.
Flag:
Check that the changed owner still handles reachable empty, loading, and missing optional-data states, and that primitive wrappers preserve accessible semantics and the public controlled-state contract. Report the actual lost behavior or explicit rule violation; use the package testing policy when assessing regression coverage.
- Refactors of existing navigation, sidebar, dropdown, webapp list, or app-switching UI that do not preserve behavior-sensitive interactions such as expand/collapse arrows, hover persistence, pin/delete controls, routing, keyboard/focus handling, or open-state ownership.
- Components that mix data fetching, mutation side effects, popup state, form validation, layout, and row rendering without a clear owner.
- Generic components with many boolean props that encode one feature's workflow.
- A shared component that imports feature-specific copy, routes, or API contracts.
- A feature component that accepts pre-rendered fragments only to avoid placing ownership correctly.
- A child component that receives both raw server data and separately derived flags for the same concept.
- A wrapper that changes accessible semantics of the primitive it wraps.
- A component that exposes controlled props but still keeps a competing private state for the same value.
- A component that cannot render empty, loading, or missing optional API fields without caller-side preprocessing.
[Interactions]: ../../how-to-write-component/references/interactions.md
[Ownership]: ../../how-to-write-component/references/ownership.md
[Runtime]: ../../how-to-write-component/references/runtime.md
[State]: ../../how-to-write-component/references/state.md
When existing components already own interaction logic, prefer reusing or extending them. If a refactor is necessary, preserve the old interaction contract and add or update focused tests for changed behavior.
## Props And Types
Flag:
- Declaration or export rewrites made only for stylistic uniformity, without changing an owned behavior or contract.
- Named `Props` types for trivial one-off props where inline typing is clearer.
- Props named by UI implementation instead of domain/API role.
- API data converted too early or under a generic name that breaks traceability.
- Callers duplicating fallback checks that the lowest rendering component already handles.
Do not flag `FC`, `React.FC`, function declarations, arrow functions, named exports, or default exports by syntax alone. Report them only when the chosen form causes a concrete type, lifecycle, export, framework, or enforced package-contract defect.
## Effects
Flag effects that:
- Transform props/state for rendering.
- Copy one state value into another representing the same concept.
- Handle user actions that belong in event handlers.
- Reset state from props when a keyed reset, stable ID, or render-time derivation would work.
- Fetch data that belongs in framework APIs or TanStack Query.
If an effect remains, it must synchronize with a named external system: browser API, subscription, timer, analytics-on-visibility, non-React widget, or imperative DOM integration.
## State Modeling
Flag:
- Storing derived booleans, disabled flags, default tabs, or loading labels that can be calculated from current query/feature state.
- Local state used to fake server data or generated contract fields.
- UI state persisted to localStorage when it is live app state.
- Feature-local mock shells wired to unrelated existing APIs before the real API is confirmed.
Prefer render-time derivation. Keep true local state for user choices, transient input, controlled popups, and feature UI state that has no server source.
## Navigation
Flag:
- Imperative router navigation for ordinary links.
- Button semantics used for navigation.
- Navigation state hidden in component state when URL state is required for shareable filters, tabs, or pagination.
Use `Link` for normal navigation. Use router APIs for mutation success, guarded redirects, command flows, or form submission side effects.

View File

@@ -1,28 +1,74 @@
# Data, Query, And Contract Review
# Data, Query, And Contract Rules
[Data and queries] owns generated-client, Query options, mutation/cache, imperative access, SSR, authentication, and tenant rules. [State ownership] owns URL state and persistence. Read only the reference needed by the diff; reading a shared reference does not activate its implementation skill.
Use these rules for generated contracts, TanStack Query, mutations, auth/SSR boundaries, URL state, and client persistence.
## Generated Contracts And Query Conventions
## Generated Contracts
Review explicit team conventions as contracts, including direct generated options, `skipToken` for missing required input, and shared cache policy. A lint rule may enforce part of a convention; neither a green lint result nor a suppression proves the full contract is satisfied.
Flag:
- Establish whether the changed call belongs to a new or migrated surface and whether the generated operation is ready. Check deprecated markers, schema shape, and the real UI consumer before prescribing a migration.
- Distinguish a pass-through wrapper from a feature hook with actual orchestration. Check whether an independent execution condition or Promise composition justifies the documented query/mutation exception.
- Trace generated input and output types through their boundaries. Identify the exact DTO mirror, field widening, placeholder input, or lost intentional empty value when reporting a violation.
- Check whether a local mutation callback owns feature feedback or replaces shared invalidation, retry, or cache defaults. Match optimistic changes to the current list/detail owner.
- New legacy service/helper wrappers around generated `queryOptions()` or `mutationOptions()`.
- Continuing to use deprecated contract operations when a ready generated contract exists.
- Assuming a generated file means an operation is ready without checking deprecated markers, schema shape, and the actual UI consumer.
- Re-declaring API DTOs in components.
- Adding compatibility layers instead of migrating the pointed line and deleting the old layer.
## Imperative Access And SSR
Backend Pydantic and OpenAPI schemas own API shape. Generated clients and schemas under `packages/contracts/generated/*` are authoritative at frontend boundaries and use the `{ params, query?, body? }` input shape.
- Check freshness, projection, retries, and the caller's execution condition separately against [Data and queries]. Trace Promise ownership to distinguish an awaited hard gate from soft prefetching with an explicit failure owner.
- For Server Components, identify who renders the data and who may revalidate it. Check dehydration, the same-key client consumer, error handling, and the intended Suspense/server-rendered-content contract when the diff changes streaming.
- For auth, setup, roles, branding, or availability, trace authoritative data and the loading/fallback path. A static redirect or placeholder value cannot stand in for a request-dependent decision.
## Queries
## Tenant, URL, And Persistence Boundaries
Flag:
- Trace the current workspace-switch flow and cache lifetime before reporting missing identity in a query key. Verify backend meaning before treating `workspace_id` and `tenant_id` as interchangeable.
- For URL and storage changes, identify whether the value is shareable navigation state, live app state, a one-shot signal, or a low-frequency preference. Apply [State ownership] to that category and verify its write/reset owner.
- `enabled` used to hide missing required input instead of `input: skipToken`.
- Fake fallback IDs or placeholder inputs used to force a query to run.
- Query results copied into local state for rendering.
- Shared query behavior such as invalidation, stale defaults, or retry rules reimplemented at call sites.
- `prefetchQuery` treated as a hard gate or as returning data/errors to the caller.
Report the violated rule and applicable scope or the concrete failing path. Do not invent runtime impact when the finding is a team-convention violation.
Use `useQuery(consoleQuery.xxx.queryOptions(...))` or `useQuery(marketplaceQuery.xxx.queryOptions(...))` directly unless a feature hook performs real orchestration.
[Data and queries]: ../../how-to-write-component/references/data.md
[State ownership]: ../../how-to-write-component/references/state.md
## Mutations
Flag:
- Deprecated `useInvalid` or `useReset`.
- `mutateAsync` used without a need for Promise semantics.
- Awaited mutations without `try/catch`.
- Components owning shared cache invalidation that belongs in query defaults.
- Optimistic updates that do not match current list/detail ownership.
Use generated `mutationOptions()` directly when possible. Put shared cache behavior in `createTanstackQueryUtils(...experimental_defaults...)`.
## SSR, Auth, And Route Boundaries
Flag:
- Request-time auth, setup, workspace role, or tenant decisions moved into static `next.config redirects()`.
- Dynamic role gates depending on `workspaces.current` implemented as static path redirects.
- Authorization logic depending on soft `prefetchQuery`.
- Removing a client fallback before server API unavailable behavior is defined.
- Global placeholder query contracts introduced to solve a route-local Suspense issue.
- Branding-sensitive UI reading placeholder defaults without checking pending/placeholder state.
Separate hard gates from soft prefetches. `fetchQuery` can be a server decision boundary; `prefetchQuery` is cache warmup.
## Workspace And Tenant
Flag:
- Treating workspace switch as ordinary CRUD invalidation when the current app flow performs server switch plus full reload.
- Query keys that omit workspace/tenant identity when the query truly varies by workspace and no full reload boundary applies.
- Mixing `workspace_id` and `tenant_id` without tracing the current backend/API contract.
Current Dify workspace switch should be reviewed as a tenant cache boundary first.
## URL State And Local Storage
Flag:
- Shareable filters, tabs, pagination, selected panels, or search state hidden only in component state.
- One-shot navigation signals modeled as subscribed persistent state.
- Live app state stored in localStorage.
- Direct `window.localStorage`, `globalThis.localStorage`, or raw storage calls in app code.
- High-frequency interaction state persisted on every change instead of on commit/settle.
Use URL state for shareable UI state, feature/Jotai/store state for live UI state, and `@/hooks/use-local-storage` only for low-frequency client-only preferences, dismissed notices, and UI defaults.

View File

@@ -1,29 +1,141 @@
# Dify UI Review Routing
# Dify UI Rules
Use this reference when a review touches `packages/dify-ui/` or consumes
`@langgenius/dify-ui/*`. It routes to owner documentation; it does not redefine package
contracts.
Use these rules whenever a review touches `packages/dify-ui/` or code consuming `@langgenius/dify-ui/*`.
Read `packages/dify-ui/AGENTS.md` and the primitive implementation first, then only the matching
owner:
Before finalizing findings for those files, read the current local docs that apply:
| Review area | Canonical owner |
| ------------------------------------------ | ------------------------------------------------------------------- |
| Package boundary and document routing | `packages/dify-ui/AGENTS.md` |
| Imports, exports, public types, and anatomy | `packages/dify-ui/docs/authoring.md` |
| Button or icon-only actions | `packages/dify-ui/src/button/README.md`, `packages/dify-ui/src/icon-button/README.md` |
| Compound inputs | `packages/dify-ui/src/input-group/README.md` |
| Forms and field semantics | `packages/dify-ui/docs/forms.md` |
| Selection and typed values | `packages/dify-ui/docs/selection.md` |
| Portals, layers, and floating surfaces | `packages/dify-ui/docs/overlays.md` |
| Tailwind and radius tokens | `packages/dify-ui/docs/styling.md` |
| Package tests and Storybook | `packages/dify-ui/docs/testing.md` |
- `packages/dify-ui/README.md`
- `packages/dify-ui/AGENTS.md`
- `web/docs/overlay.md` for floating UI
- `packages/dify-ui/src/<primitive>/index.tsx` for the primitive being changed or consumed
For consumer code under `web/`, also read `web/AGENTS.md` for application-owned reuse policy and
`packages/dify-ui/README.md` for the available public subpaths.
## Package Boundary
Treat the implementation, public types, tests, and stories as evidence for the documented
contract. If they disagree, identify the actual owner before reporting a finding. Read current
official Base UI documentation and installed type declarations for upstream-derived behavior.
Report only a reproducible contract violation or observable defect, not a preference inferred from
this routing file.
Flag in `packages/dify-ui`:
- Imports from `web/`.
- Dependencies on Next.js, i18n, ky, Jotai, Zustand, TanStack Query, oRPC, or business APIs.
- Business-specific component behavior that belongs in `web/`.
- Multiple unrelated primitives in one component folder.
`packages/dify-ui` is a primitive layer: Base UI headless components + `cva` + `cn` + Dify design tokens.
## Imports And Exports
Flag:
- Consumer imports from `@langgenius/dify-ui` without a subpath.
- Missing `package.json#exports` entry for a new primitive.
- Internal package imports using workspace subpaths instead of relative paths.
- Exported props using internal-only types that consumers cannot import from the component subpath.
- Canonical primitive boundaries or their associated public types using a redundant `Root` suffix when no higher-level convenience component exists in the same subpath.
Consumers use subpath exports such as `@langgenius/dify-ui/button`.
Canonical boundaries use the primitive name and matching public types (`Select` / `SelectProps`). Keep `Root` only to distinguish a low-level anatomy root from a higher-level convenience component (`CheckboxRoot` / `Checkbox`); implementation aliases should still show their Base UI source (`BaseSelect.Root.Props`).
## Props And State
Flag:
- Flattened props where related values need a discriminated union, such as `value` / `defaultValue`, `multiple` / `value`, or `clearable` / `onChange`.
- React state used only to mirror Base UI state for class names.
- JavaScript conditional class logic for visual states that the Dify UI/Base UI primitive already exposes through `data-*` attributes or CSS variables.
- Controlled props added when uncontrolled DOM state or CSS variables would be enough.
- Thin wrappers that rename Base UI parts without adding semantics.
- Generic Base UI selection primitives wrapped without preserving their value generics, such as `Select.Root<Value, Multiple>`, `RadioGroup<Value>`, or `Radio.Root<Value>`.
- Shared select/radio option components that type selected values as `string` while callers pass enums, unions, booleans, numbers, objects, or nullable placeholder values.
Prefer Base UI/Dify UI data attributes and CSS variables for visual state: `data-open`, `data-checked`, `data-disabled`, `data-highlighted`, `data-popup-open`, `group-data-*`, `peer-data-*`, `has-[:focus-visible]`, and primitive CSS variables such as anchor width or transform origin. Use JS conditional classes for product/business state that the primitive does not expose.
For non-string `Select` and `RadioGroup` values, prefer explicit domain generics at the root and at child value carriers. JSX children do not inherit the parent generic, so `RadioGroup<PromptMode>` should compose with `Radio<PromptMode>`, `RadioItem<PromptMode>`, or option values from a typed collection. For `Select`, prefer the Base UI `items` collection pattern for typed value-to-label rendering, and flag string coercion helpers used only to recover display labels.
## Forms
Flag:
- Form-like UI using unrelated `Input` and `Button` pieces without a submit boundary.
- Text-like fields not composed through `Field`, `FieldLabel`, and `FieldControl` when using Dify UI form semantics.
- Select fields using `FieldLabel` instead of `SelectLabel`.
- Slider fields using a generic label instead of `SliderLabel`.
- Checkbox/radio groups missing `Fieldset` and `FieldsetLegend`.
- Field errors or descriptions rendered without `FieldDescription` / `FieldError` relationships.
`Form` is the submit boundary. Dify UI form primitives are not a form state-management framework; business validation and schema-driven behavior belong in `web/`.
## Overlay Contract
Flag:
- Legacy web overlay imports in new or modified code.
- Manual portals around Dify UI overlay primitives.
- Call-site `z-*` overrides on overlays.
- Missing root `isolation: isolate` assumptions when debugging overlay stacking.
- Repeated backdrop, z-index, or portal chrome at call sites.
- Tooltip used for infotips, long text, or interactive content.
All Dify UI body-portalled overlays use `z-50`. Toast uses `z-60`. DOM order handles stacking between overlays.
## Primitive Selection
Flag:
- `Tabs` used for simple mode/filter/view selection where `SegmentedControl` is the semantic primitive.
- `SegmentedControl` used where `tablist` / `tabpanel` semantics are required.
- `Select` used for searchable or free-form input.
- `Combobox` used for unrestricted search text where no selected option is remembered.
- `Autocomplete` used for closed-list selection.
- Tooltip or PreviewCard used for content that must be reachable on touch or by screen readers.
Use:
- `Autocomplete` for free-form text with optional suggestions.
- `Combobox` for searchable selected values from a collection.
- `Select` for closed, scannable option sets.
- `Popover` for infotips, help text, rich content, or interactions.
## Bad Usage Patterns To Flag
Flag:
- Manually recreating UI behavior or chrome already owned by `@langgenius/dify-ui/*` or `web/app/components/base/*`, such as buttons, inputs, toggle groups, popovers, dropdown menus, alert dialogs, switches, avatars, scroll areas, toasts, borders, focus states, disabled states, segmented controls, or existing feature components.
- Styling a raw Base UI primitive directly in `web/` when a Dify UI primitive exists.
- Wrapping a Dify UI primitive in a feature component that hides its label, error, disabled, or focus contract.
- Replacing a semantic primitive with a generic `div` plus classes to match a screenshot.
- Using `Tooltip` because it is visually convenient when the content is actually help text or needs touch access.
- Adding a `z-*` override to make a child popup appear over a parent dialog.
- Adding a new app-level wrapper around Dialog, Drawer, Popover, Select, or Combobox that repeats portal/backdrop/positioner logic.
- Using dify-ui `Input` as a drop-in replacement for legacy inputs that include search, clear, copy, unit, localized placeholder, or number normalization behavior.
- Building a form row from loose text and controls instead of the matching Field/Form primitives.
- Adding component state only to style `data-open`, `data-checked`, `data-disabled`, or highlighted states that Base UI already exposes.
- Passing booleans down only so children can toggle classes already expressible with primitive `data-*` selectors.
## Tokens, Radius, And Styling
Flag:
- `radius-*` class names.
- Custom Tailwind `borderRadius` extension for Figma radius values.
- Generic colors where semantic Dify tokens exist.
- Hardcoded design values where Dify tokens, component variants, or documented Figma radius mappings exist.
- `!` important modifiers used to fight primitive styles instead of fixing the variant, selector, or component composition.
- Manual class strings that duplicate primitive variants.
- `min-w-(--anchor-width)` on picker popups when it defeats viewport clamping.
Use the Figma radius mapping from `packages/dify-ui/README.md`; for example `--radius/sm` maps to `rounded-md`, and `--radius/md` maps to `rounded-lg`.
Use `!` only for a tightly scoped compatibility override after confirming the primitive API, data attributes, and selector structure cannot express the state.
## Focus Details
Flag focus rings attached to the wrong element. For example, Base UI `Slider.Thumb` focuses an internal `input[type=range]`, so the visible thumb wrapper needs `has-[:focus-visible]` rather than direct wrapper `focus-visible`.
## Custom SVG Icons
Flag:
- New generated React icon components or JSON files under `web/app/components/base/icons/src/...` for custom SVG icons.
- Custom SVG icons consumed outside the Tailwind `i-custom-*` icon class pipeline.
- Generated `packages/iconify-collections/custom-*/icons.json` diffs where unrelated existing icons lost or changed intrinsic `width` or `height`.
New custom SVG icons belong in `packages/iconify-collections/assets/...`. Regenerate with `pnpm --filter @dify/iconify-collections generate`, validate with `pnpm --filter @dify/iconify-collections check:dimensions`, and consume the generated icon with Tailwind `i-custom-*` classes.

View File

@@ -71,6 +71,8 @@ Flag:
## React Flow
Use [Dify invariants] for React Flow node/edge consumption and provider availability in RAG Pipe template rendering. Callback-only reads or mutations can use `useStoreApi`.
For workflow React Flow components, keep this Dify-specific rule:
[Dify invariants]: dify-invariants.md
- UI consumption should use React Flow hooks such as `useNodes` / `useEdges`.
- Callback-only reads or mutations can use `useStoreApi`.
- Node components under `web/app/components/workflow/nodes/[nodeName]/node.tsx` must not depend on workflow stores that are absent in RAG Pipe template rendering.

View File

@@ -1,6 +1,6 @@
# Testing Review Rules
Use `web/docs/test.md` for tests owned by `web/` and `packages/dify-ui/docs/testing.md` for tests owned by Dify UI. These owners define their test boundaries, environments, and checks; this reference only adds review questions.
`web/docs/test.md` is the canonical frontend testing policy. Use this file only to translate that policy into review findings.
## Request Missing Tests When Risk Justifies Them
@@ -9,7 +9,6 @@ Flag missing coverage when a change alters a reachable contract such as:
- User interaction, navigation, form submission, validation, or permissions.
- Query or mutation behavior, URL state, persistence, or one-shot signals.
- Loading, error, empty, and recovery states that users can encounter.
- A hidden surface whose close-and-reopen behavior changes whether in-progress state resets or persists.
- Accessibility-critical labels, keyboard flow, focus, disabled state, or overlay behavior.
- A regression-prone business rule or bug fix that can be reproduced through a public boundary.
@@ -30,7 +29,8 @@ Flag tests that:
- Prefer semantic queries and accessible names.
- Prefer real feature components when integration semantics matter.
- Check mocks against the owning package's policy; allowed mocks must preserve the public contract and leave the behavior under review real.
- Allow intentional child or provider mocks when setup would dominate the test and that boundary is covered independently.
- Do not accept semantically inaccurate mocks of Dify UI or legacy base primitives.
- Require a real-browser or visual verification plan when `happy-dom` cannot represent the risk.
Treat test quality, determinism, and regression value as the review criteria. Do not use test count or coverage percentage as a proxy for quality.

View File

@@ -5,9 +5,12 @@ description: Use when writing or changing Vitest or React Testing Library tests
# Frontend Testing
Read the testing policy for the package that owns the changed tests:
`web/docs/test.md` is the single policy owner. Read it before changing frontend tests; this skill adds no separate requirements.
- `web/`: `web/docs/test.md` owns Web test boundaries, environments, and commands.
- `packages/dify-ui/`: `packages/dify-ui/docs/testing.md` owns primitive test boundaries, environments, Storybook, and commands.
1. Identify the observable contract and regression risk.
2. Choose the smallest boundary that includes the behavior owner.
3. Establish the failing case first when practical, then implement one coherent scenario.
4. Run the focused spec before the affected suite and relevant static checks.
5. Report the behavior verified and any remaining browser, visual, or end-to-end risk.
This skill adds no parallel policy or check sequence. Follow the selected owner's requirements, including its validation commands. Recommend deleting low-value tests as readily as adding missing behavior coverage, and report the contract verified and any material verification gap.
Recommend deleting low-value tests as readily as adding missing behavior coverage. Use `web/docs/test.md` for policy and Web commands; use the `packages/dify-ui/README.md` Development section for Dify UI commands.

View File

@@ -1,62 +1,38 @@
---
name: how-to-write-component
description: Use when implementing, refactoring, or architecture-reviewing React/TypeScript components whose ownership, state lifetimes, data flow, props, feature boundaries, effects, or interaction surfaces need redesign. Do not use for bug, regression, security, or accessibility audits; test-only work; copy-only edits; or styling-only changes.
description: Use when implementing or refactoring React/TypeScript components and the task requires decisions about component ownership, feature boundaries, state, data flow, effects, or interaction ownership. Do not use for review-only requests, test-only work, copy-only edits, or styling-only changes.
---
# React Component Architecture
# How To Write A Component
Use this skill to make component ownership and lifecycle explicit before implementation. Classify every stateful value on three independent dimensions:
Use this skill to route component architecture decisions to its bundled references. Read only the references required by the current change.
1. **Owner/source:** a component, parent boundary, URL/framework API, TanStack Query, or feature workflow.
2. **Graph role:** primitive input, query/mutation node, derived business fact, or workflow command.
3. **Lifecycle:** instance isolation, persistence, and reset requirements.
## First Decisions
Do not force one label to answer all three dimensions.
Keep display state, submit-only fields, and owner-local queries or mutations with the component or DOM that owns their complete lifecycle. A value enters the feature state graph only when it crosses an ownership boundary, participates in shared derivation or coordination, or must outlive/reset independently of its current owner. Driving an owner-local query or command alone is not sufficient. In this repository, use feature-local Jotai as the default representation once a value enters that graph.
## Operating Modes
Choose the mode from the user's requested outcome:
- **Architecture audit:** Perform a read-only, root-to-leaf analysis and propose the target ownership model. Read [`references/audit.md`][audit] and [`references/ownership.md`][ownership].
- **Refactoring design:** Perform the architecture audit, then provide target contracts, migration slices, and a verification strategy without modifying code. Read [`references/audit.md`][audit] and [`references/ownership.md`][ownership].
- **Implementation:** Establish the affected owner map, state graph, props edges, and reset boundaries before editing. Implement one coherent vertical slice at a time, then run the final audit in [`references/audit.md`][audit]. Read [`references/ownership.md`][ownership].
- **Bug or regression review:** Use the frontend code-review workflow unless the user explicitly requests this skill's architecture model as an additional lens.
| Question | Default | Promote only when |
| --- | --- | --- |
| Where should code live? | In the product workflow, route, or feature owner. | Several verticals need the same stable contract. |
| Who owns state and handlers? | The lowest visual owner that consumes them. | A parent coordinates one workflow or consistent snapshot. |
| Should state enter Jotai? | Keep component and form state local. | Siblings need one source of truth or scoped workflow persistence. |
| Who owns URL state? | Next.js route APIs and `nuqs`. | Atoms require a read-only route-identity bridge. |
| Who owns remote state? | TanStack Query at the lowest consumer. | Atom state drives the query or shared derivations consume it. |
| Is a wrapper needed? | Use the primitive or direct code. | The wrapper owns behavior, validation, state, or semantics. |
| Is an Effect needed? | Derive during render or handle the user action. | A named external system must be synchronized. |
## Topic Routing
After selecting the operating mode, read only the topic references required by the change:
- Jotai, graph-entry decisions, form drafts, shared client state, route identity, URL state, isolation/reset, or persistence: read [`references/state.md`][state].
- Generated contracts, nullable API data, Query, mutations, SSR, authentication, or workspace state: read [`references/data.md`][data].
- A feature graph that combines Jotai isolation with TanStack Query atoms or commands: read both [`references/state.md`][state] and [`references/data.md`][data].
- Hotkeys, focus, dialogs, menus, popovers, or other secondary surfaces: read [`references/interactions.md`][interactions] and the overlay guide it references when applicable. Also read [`references/state.md`][state] when the surface owns a draft or session state.
- Component moves, module boundaries, props, types, or owner placement: read [`references/ownership.md`][ownership].
- Jotai, form drafts, route identity, URL state, or persistence: read [`references/state.md`][state].
- Generated contracts, nullable API data, Query, mutations, SSR, auth, or workspace state: read [`references/data.md`][data].
- Hotkeys, focus, dialogs, menus, popovers, or other secondary surfaces: read [`references/interactions.md`][interactions] and the overlay guide it references when applicable.
- Effects, navigation, memoization, preloading, or render cost: read [`references/runtime.md`][runtime].
## Common Workflow
## Workflow
Follow this order so component splitting does not precede ownership decisions:
1. **Boundary and evidence:** identify the route, tab, workflow, or action surface that owns the behavior and lifetime. Inspect nearby implementation, tests, sibling state files, and established feature boundaries.
2. **Current model:** identify generated data contracts and authoritative external inputs, then inventory state, queries, mutations, derived facts, commands, refs, and meaningful props edges.
3. **Target graph and owners:** decide explicitly which values remain local and which enter the feature graph. Define authoritative bridges, graph nodes, component owners, and scope/reset boundaries before creating files or wiring components.
4. **Re-cut the tree:** treat current files as evidence rather than target constraints. Keep, split, merge, remove, rename, promote, or demote components based on ownership, lifecycle, behavior, and independently changing visual regions.
5. **Contracts and interactions:** place data, loading, empty, error, and handlers at the lowest real consumer. Cross true owner boundaries only with stable identity, a small immutable snapshot, placement, or a named command. Give forms and secondary surfaces explicit lifecycle owners.
6. **Finish and verify:** remove copied state and unnecessary Effects, wrappers, memoization, or nullable coercion. Rerun the final owner, graph, props-edge, and reset audit, then verify observable behavior at the narrowest sufficient boundary.
## Scope And Verification
In architecture-audit and refactoring-design modes, do not modify code. In implementation mode, complete equivalent ownership fixes inside the audited feature when the target contract depends on them, but do not expand into unrelated features. Incremental slices may retain documented temporary edges; the final requested slice must pass the completion gate in [`references/audit.md`][audit].
Run the checks documented by the owning package: `web/docs/test.md` or `web/docs/lint.md` for Web, and `packages/dify-ui/docs/testing.md` for Dify UI.
[audit]: references/audit.md
## Tailwind CSS
- Write canonical Tailwind v4 classes; prefer canonical utilities over equivalent arbitrary values.
- Common forms include `w-105` instead of `w-[420px]`, `px-2.25` instead of `px-[9px]`, `bg-linear-to-b` instead of `bg-gradient-to-b`, `wrap-break-word` instead of `break-words`, and `field-sizing-content` instead of `[field-sizing:content]`.
1. Identify the behavior owner and the public contract being changed.
2. Read the nearby implementation, tests, and only the routed skill references.
3. Implement one coherent vertical slice. Do not expand into equivalent patterns elsewhere unless the current contract cannot be completed without them.
4. Verify observable behavior at the narrowest sufficient boundary, then run the checks documented by the owning package: `web/docs/test.md` or `web/docs/lint.md` for Web, and the `packages/dify-ui/README.md` Development section for Dify UI.
[data]: references/data.md
[interactions]: references/interactions.md

View File

@@ -1,64 +0,0 @@
# Component Architecture Audit
Read this document for architecture audits, refactoring designs, and the final ownership verification after an implementation refactor.
## Audit Procedure
For a named component, use it as the audit root. For a refactoring request, use the nearest route, page, feature entrypoint, or behavior owner whose state contract is changing.
1. Build the complete locally owned rendered component tree from the root to every leaf. Follow imported feature components, conditional branches, lists, portals, dialogs, drawers, popovers, and secondary surfaces. Stop at third-party primitives, Dify UI primitives, or an explicit stable feature boundary, but record the props passed across the final edge.
2. At every component, inventory React state and reducers, form and uncontrolled DOM state, URL and route state, Context/Jotai/store reads and writes, queries, mutations, refs that hold workflow or lifecycle state, custom-hook results, render-time derived facts, and workflow commands or handlers.
3. For every value, record its authoritative owner/source, graph role, declaring component, consuming branches, lifetime and reset/isolation requirements, and whether it is local, shared, forwarded, mirrored, or persisted.
4. Inspect every meaningful props edge. Mark each prop as consumed, forwarded unchanged, renamed, recomputed, mirrored into local state, or paired with lifecycle fields such as `data/pending/error/retry` and `open/onOpenChange`.
5. Draw the current state graph in dependency order and assess ownership only after the complete paths are mapped. Group related problems at the highest incorrect owner instead of repeating the same issue for every descendant.
6. Give every current local component an explicit target disposition: keep, split, merge, remove, rename, promote to an owner, or demote to presentation. Map every current state or workflow owner to a target component and reset boundary.
7. Redraw the target rendered tree independently of the current file layout. Split only when state lifetime, interaction lifecycle, behavior ownership, or an independently changing visual region establishes a real boundary. Merge or remove components that only forward, rename, or obscure a single owner's contract.
8. For refactoring design or implementation planning, define ordered migration slices with explicit state-file changes, component moves/splits/merges, observable verification boundaries, and intentionally temporary props edges.
A rename, facade wrapper, props bag, Context, or provider around the same switchboard does not count as a boundary redesign. Correctness bugs may illustrate an ownership defect, but architecture-audit mode does not prioritize or severity-rank bug findings.
## Audit Output
Cover the following information. Merge sections or omit empty tables when the review is small, but do not omit applicable values, paths, edges, owners, or reset boundaries:
1. **Review boundary:** root component, locally owned rendered paths, and stopping boundaries.
2. **Rendered component tree:** every local branch and secondary surface followed.
3. **State inventory:** owner/source, graph role, graph-entry decision, declaration, consumers, and lifetime/reset needs for every stateful value, query, mutation, ref, custom-hook result, derived fact, and command.
4. **Props-edge ledger:** every meaningful parent-child edge and how each prop is treated.
5. **Current architecture:** state graph, component owners, and ownership assessment.
6. **Component dispositions:** the target disposition of every current local component and the destination of every current state/workflow owner.
7. **Target architecture:** target rendered tree, state graph, owners, component contracts, and reset boundaries.
8. **Migration slices:** required for refactoring design; include them in an audit when they materially clarify how to reach the target.
Use compact tables when they make full accounting easier to verify:
| Value | Source/role | Declared at | Consumers | Props path | Lifetime/reset | Recommended owner |
| --- | --- | --- | --- | --- | --- | --- |
| Edge | Prop | Treatment | Real consumer | Assessment |
| --- | --- | --- | --- | --- |
| Current component | Current responsibilities | Target disposition | Target owner(s) | Reason |
| --- | --- | --- | --- | --- |
| Target component | Owned state/workflow | Children | Public contract | Reset boundary |
| --- | --- | --- | --- | --- |
## Implementation Completion Gate
Repeat the audit against the final code. Do not describe an implementation or refactor as complete until all applicable checks pass:
- every locally owned rendered path, meaningful props edge, and stateful value is represented in the component tree, inventory, or ledger;
- every current local component has a target disposition, and every current state/workflow owner maps to an implemented component and reset boundary;
- every multi-consumer graph value has one authoritative bridge or graph node and a named owner;
- route or URL identity is read directly by a single consumer or bridged once, not also threaded through descendant props;
- query keys, observer objects, `refetch`, loading/error groups, and cache invalidation mechanics do not cross a boundary unless the child owns the query surface;
- no page, hook, provider, or Context mainly destructures, renames, and redistributes another state machine;
- components consume focused fields, named facts, or domain commands instead of rebuilding shared business conclusions;
- query and mutation atoms remain cache-shared, while only resettable workflow primitives or boundary snapshots are scoped;
- a feature scope that depends on application state still observes parent/global atom updates; a fresh `Provider` is used only when full-store isolation is an explicit requirement;
- scoped-state verification covers parent-store interoperation, sibling-instance isolation when multiple instances are supported, and semantic-identity reset;
- every remaining multi-layer prop edge is justified as stable identity, an immutable display snapshot, placement, or a named cross-boundary command;
- the final rendered tree, state graph, owner map, contracts, and reset boundaries match the implemented code rather than only the initial design.
If the user requests incremental commits, a slice may intentionally leave documented edges for a later slice. The final requested slice must still pass this gate.

View File

@@ -8,57 +8,35 @@ Read this document when a component consumes generated contracts, nullable API v
- Backend Pydantic and OpenAPI schemas own API shape. Follow the generated `{ params, query?, body? }` input shape; when it is wrong, fix the backend schema and regenerate `packages/contracts/generated/*`.
- Do not hand-write DTO mirrors, widen generated fields or enums, edit generated output, or add a parallel frontend status layer unless it models product state absent from the API.
- Check deprecated markers, schema shape, and the actual consumer before assuming that a generated operation is ready to use.
- When a ready generated operation exists for the changed call, migrate deprecated operations and remove the replaced layer instead of adding compatibility wrappers.
- Normalize only at real boundaries such as user input, search, URL params, filenames, DOM IDs, or a required legacy adapter.
- Preserve `null`, `undefined`, and intentional empty strings until the final boundary. Do not use `value || undefined` when an empty string means clearing a field.
- Build required values in the branch that proves them. Do not use truthiness filters, non-null assertions, or placeholders to conceal missing required input or discard valid `0`, `false`, or empty-string values.
- Build required values in the branch that proves them. Avoid `filter(Boolean)`, truthiness filters, non-null assertions after filters, and placeholder values used only to satisfy types.
## Queries
- Use generated options directly with `useQuery(consoleQuery.xxx.queryOptions(...))`, `marketplaceQuery`, or the equivalent generated client.
- If query input comes from atom state, keep it in `atomWithQuery`; do not unwrap the atom in a component solely to call `useQuery`.
- Keep owner-local queries in `useQuery` when their pending, error, and data state serve only that component. Promote them to query atoms only when atom input drives them or other graph nodes consume their result.
- When a query supplies several sibling owners, shared derived facts, or workflow commands, model it as a graph node with field selectors. Do not keep it in a page and pass its data, query key, observer methods, and status fields back down as a deconstructed query object.
- For missing required input, branch the whole generated input with `skipToken`. Add `enabled` only for an independent execution condition; do not put `skipToken` inside a placeholder payload or coerce IDs to empty strings.
- Return generated `queryOptions()`, `infiniteOptions()`, or `mutationOptions()` directly from TanStack Query atoms. Pass supported options into the generated call instead of spreading into a parallel object.
- For the same logical request, preserve key, input, operation, and result contracts. Share the exact options between prefetch and render only when transport, context, and cache policy are shared; imperative and observer freshness may differ. Do not extract option helpers merely to reuse input construction.
- In Jotai-backed components, consume field-level selectors or named facts rather than the complete query result unless observer methods such as `refetch` or an infinite-scroll field group are part of that owner's contract.
- Keep observer methods inside the component that owns the query surface or behind a named graph command. A parent must not pass raw `refetch`, query keys, or invalidation details merely because a descendant action needs fresh data.
- Treat `refetch()` as an explicit execution command: it can run a query whose observer currently has `enabled: false`. Retry and refresh commands must recheck the permission, identity, or availability condition before including conditionally enabled queries.
- Cache writes and query-observer notifications may reach React through asynchronous batching, including when `atomWithQuery` observes an existing QueryClient. Tests that call `setQueryData`, invalidate, or complete a refetch must await the resulting DOM fact with `findBy*` or `waitFor`; do not assume the observer rerender is synchronous with the cache call.
- Share the exact options between prefetch and render when they represent the same request. Do not extract option helpers merely to reuse input construction.
- Avoid pass-through service hooks that only rename generated options. Keep feature hooks for actual orchestration or shared domain behavior.
## Mutations And Cache
- Use generated `mutationOptions()` directly for owner-local mutations.
- Keep a mutation in `useMutation` when pending and error state belong to one dialog or form. Use a mutation atom only when shared workflow orchestration or graph-derived state consumes it; query and mutation atoms remain unscoped.
- Do not introduce deprecated `useInvalid` or `useReset` APIs.
- Put shared invalidation, retries, and cache behavior in `createTanstackQueryUtils(...experimental_defaults...)`. In oRPC v1, caller options override defaults, including callbacks. Local callbacks may own toast, close, and navigation effects but must preserve shared cache behavior.
- Workflow-specific invalidation belongs beside the mutation/command that understands which caches become stale. Construct generated query options or keys from graph identity there instead of receiving keys from a rendering parent.
- Put shared invalidation, retries, and cache behavior in `createTanstackQueryUtils(...experimental_defaults...)`. Local callbacks may own toast, close, and navigation effects but must not replace shared cache policy.
- Prefer `mutate(...)`. Use `mutateAsync(...)` only when Promise composition is required, and catch awaited failures.
- Preserve intentional empty values and current list/detail ownership when updating data. Do not add optimistic updates without a verified owner contract.
## Prefetch And Hidden Surfaces
- Prefetch expensive secondary content from the trigger or menu-open event when it benefits the visible path. Do not mount hidden subscribers solely to warm the cache.
- Use `query` or `infiniteQuery` for imperative access. `staleTime` defines cache acceptance; `select` projects the return
value without replacing cached query-function data. Imperative queries default to no retries when `retry` is not
configured, and observer-only `enabled` does not prevent an imperative call.
- Migrate deprecated `fetchQuery`, `prefetchQuery`, `ensureQueryData`, and their infinite variants when the current `query` or `infiniteQuery` contract applies. A `staleTime` of `0` treats data as stale; a finite value accepts that freshness window; `Infinity` accepts data until invalidation; `'static'` accepts available data even after invalidation.
- `await` blocks, `return` transfers the Promise, and `void` discards its value without handling rejection. Handle
rejection before discarding a potentially rejecting Promise; use `.catch(noop)` only for intentional silence or
feedback owned elsewhere. Hard server gates await the query and preserve rejection.
- `prefetchQuery` is cache warmup, not an authorization or availability gate. Use a hard fetch boundary when the server must decide whether rendering may proceed.
## SSR, Authentication, And Workspace
- Static configuration owns path-invariant routing. Request-dependent authentication, setup, role, and tenant decisions belong to SSR or runtime decision boundaries.
- Distinguish soft SSR cache warming from authoritative decisions. Prefetched or placeholder data must not grant access or represent successful availability.
- Treat Server Components as query prefetch-and-dehydrate owners by default. Do not render or pass an imperative query
result when a browser observer can revalidate the same data unless ownership and freshness explicitly prevent drift.
- Non-blocking RSC streaming requires pending-query dehydration without redacting Next.js server errors, a
`HydrationBoundary` around the same-key client consumer, and Suspense when that content must be server-rendered.
- Never reuse tenant-scoped state after switching workspaces. Discard it at the switch boundary or isolate it by workspace identity.
- Trace the current switch flow before choosing cache handling: server switch plus full reload is a tenant boundary, not ordinary CRUD invalidation. Include workspace identity in varying query keys when no full-reload boundary applies, and trace the backend contract before interchanging `workspace_id` and `tenant_id`.
- Do not make product or authorization decisions from bootstrap defaults. Wait for authoritative data, or render an explicit loading or error state.
- Preserve an existing client fallback until server API-unavailable behavior has an explicit owner.
- Keep loading and Suspense behavior inside the feature that owns the request. Do not add fake global data merely to bypass that boundary.

View File

@@ -1,6 +1,6 @@
# Component Interactions And Overlays
Read this document when a change involves application hotkeys, focus, dialogs, menus, popovers, or other secondary surfaces. Overlay primitive selection and layering are owned by the [overlay contract].
Read this document when a change involves application hotkeys, focus, dialogs, menus, popovers, or other secondary surfaces. Overlay primitive selection and layering are owned by the [overlay guide].
## Focus And Semantics
@@ -20,15 +20,12 @@ Read this document when a change involves application hotkeys, focus, dialogs, m
## Secondary Surfaces
- Follow the [overlay contract] for primitive choice and shared mechanics. The nearest consumer `AGENTS.md` owns application-specific composite reuse policy.
- A controlled composite may emit several callbacks synchronously for one gesture, such as changing a mode and then initializing its selection. Every callback sees the same render snapshot. Compose changes through one owner command or functional state update; when later callbacks or immediate side effects require the latest draft before React rerenders, maintain an interaction-owned ref updated in the handlers and clear it at the same semantic reset boundary as the draft.
- Follow `web/docs/overlay.md` for primitive choice. Dify UI primitives are the default, with package-approved Web wrappers such as `Infotip` where the overlay guide allows them.
- Separate behavior ownership from placement ownership: the action may own trigger, open state, and menu content while the caller owns slots, offsets, and alignment.
- Keep menu and dialog surfaces as siblings when a menu command opens a dialog. Mount the dialog outside popup content.
- Treat an action surface as a state boundary when its input feeds several sibling menus, dialogs, default values, confirmation messages, or mutation parameters. Inject the smallest stable identity or snapshot once; keep the form draft local unless other owners must react to it.
- Use the [overlay contract] to determine content lifetime and preserve the Root's closing lifecycle. Follow [state ownership] for draft placement and semantic identity; portal placement alone does not locate the state owner.
- Place query subscriptions and mutation observers at the owner whose lifetime matches when they should run. Mounted-session work may belong inside content; work that must start or stop exactly with `open` needs an explicit open-state condition.
- Prefer primitive-owned open state unless another owner must observe or coordinate it. Analytics callbacks and local cleanup alone do not require a controlled root.
- Do not merge dialogs with different loading guards, reset policies, or lifetimes into one discriminated `currentDialog` state merely to reduce atom count.
- Mount controlled overlays unconditionally unless unmounting is required for performance or reset semantics. Prefer keyed or owner-local reset over conditional wrappers.
- Put query and mutation work inside dialog or alert-dialog content when it should mount only after opening.
- Prefer uncontrolled roots when the primitive can own open state. Use controlled state only for business coordination, analytics, cleanup, or explicit reset behavior.
- Do not add manual portals or call-site z-index escalation. Fix ownership and stacking structure at the shared boundary.
[overlay contract]: ../../../../packages/dify-ui/docs/overlays.md
[state ownership]: state.md
[overlay guide]: ../../../../web/docs/overlay.md

View File

@@ -1,6 +1,6 @@
# Component Ownership And Modules
Read this document when auditing, adding, moving, splitting, or refactoring React components or feature modules.
Read this document when adding, moving, splitting, or refactoring React components or feature modules.
## Vertical Modules
@@ -12,42 +12,28 @@ Read this document when auditing, adding, moving, splitting, or refactoring Reac
## Component Ownership
- Before declaring state, a query, or a workflow hook in a page or parent, identify the direct descendant branches that consume each returned value.
- If only one branch consumes a value, declare it in the lowest owner in that branch whose mounted lifetime matches the required persistence. The parent may pass stable identity or the smallest boundary input, but must not own child state merely to construct props.
- A parent may own a value when it must intentionally survive the local owner's unmount, or when several sibling branches require one live snapshot and the parent genuinely derives or coordinates submission, selection, batch behavior, navigation, lifecycle, loading, or errors. If it only destructures, renames, and forwards fields, it is a switchboard rather than an owner; put the shared workflow in a feature-local state graph so each surface consumes only its named facts and commands. Use a provider only when it becomes the authoritative input, scope, or external-dependency owner.
- A provider is a real boundary only when it establishes authoritative input, scope/isolation, or a stable external dependency. A provider that calls a large hook and repackages its return value into Context remains a switchboard.
- A page or feature root may wire route identity, providers, layout, navigation, and genuine cross-surface coordination. It must not call a child-specific state or query hook merely to assemble that child's props.
- Keep child contracts at the ownership boundary: stable domain identity, a small immutable snapshot, placement options, or named cross-boundary commands. Do not pass an internal state machine as separate `data`, `pending`, `error`, `retry`, `open`, setter, and callback props when the parent does not use them, and do not hide the same fan-out in a props bag or hook result object.
- Repeated TanStack Query hooks for the same key and input in Client Component siblings under one QueryClient share that cache. Separate Server Component QueryClients do not, so request-level deduplication needs an identified request-local cache or verified framework or transport owner.
- Treat parent input according to what the child boundary does with it:
- If the child only renders or performs a light local decision, pass the snapshot as props. It does not become a new state owner.
- If the child builds queries, dialogs, mutations, derivations, commands, or a reset lifecycle around a stable identity or snapshot, give that boundary a feature-local state file. This does not by itself require a new module or directory.
- Pass stable domain identity or the smallest sufficient action snapshot across boundaries. Do not copy props into atoms merely to avoid passing them, and do not pass raw server data together with separately derived flags for the same concept.
- One pass-through layer is acceptable for stable identity and placement. It is not permission to relay workflow state and handlers through an unrelated component.
- Route identity may pass once from a framework route into its feature boundary. If multiple descendants, queries, facts, or commands need it, bridge it into the feature graph and stop passing it as props.
- A query snapshot may cross once as immutable display input. Query keys, observer methods, retry/loading/error groups, and invalidation mechanics belong to the query surface or feature graph and must not be decomposed into props.
- Revisit prop forwarding when intermediate components obscure the behavior owner; keep clear data flow rather than introducing shared state merely to avoid passing props.
- Put state, data access, loading, empty, error, and handlers in the lowest visual owner that uses them.
- Keep coordination in a parent only when it needs one consistent snapshot or coordinates submission, shared selection, batch behavior, navigation, or cross-section loading and errors.
- Repeated TanStack Query calls in siblings are acceptable when each sibling independently consumes the data; the cache already deduplicates requests.
- Pass stable domain identity across boundaries. Do not pass raw server data together with separately derived flags for the same concept.
- One pass-through prop layer is acceptable. Repeated forwarding means ownership should move closer to the consumer or into feature-scoped shared state.
- Do not replace prop drilling with one large view-model hook. Move each query, derived value, and handler to the concrete owner that consumes it.
- Keep source selection, defaults, validation, dirty checks, and payload shaping beside the workflow that owns submission.
## Boundaries
- Prefer reusing or extending the component that already owns an interaction over rebuilding that behavior in a parallel component.
- State-heavy wizards, drawers, modals, and secondary workflows can form a small vertical surface with an entrypoint, optional feature-local state, and shallow owners matching real visual regions.
- The entrypoint owns route integration, provider wiring, placement, and open-state coordination. A content or session owner keeps state scoped to that mounted surface.
- Judge hook lifetime by the component that declares the hook and the primitive's mount contract, not only by where its rendered controls appear in JSX.
- The entrypoint owns route integration, provider wiring, close behavior, and mounting. Composition owners handle workflow branches; the closest visual owner handles section branches.
- Separate hidden dialogs, dropdowns, and popovers into small local owners when their content obscures the parent flow.
- Keep cohesive forms, menu bodies, and one-off helpers local unless they have their own state, reuse, or semantic boundary.
- Extract components or hooks when they clarify ownership or hide a cohesive implementation; keep logic local when extraction only shortens the file or relocates the same coordination.
- Avoid wrapper components and wrapper DOM that only rename props, pass children through, or hide the real primitive. A wrapper must own behavior, validation, state, accessibility, layout, or library integration.
- Keep feature workflows out of shared components: do not encode one feature through many boolean props or import its copy, routes, and API contracts into a generic component. Do not pass pre-rendered fragments merely to avoid assigning the behavior to its owner.
- Loading states for page sections, cards, lists, tables, forms, and drawers should use skeletons scoped to the loaded content. Reserve spinners for small inline busy indicators.
## Components And Types
- Choose component declaration and export forms from the actual component contract, framework requirements, and enforced package rules. Existing style is context, not authority; do not rewrite unaffected code solely to normalize `FC`, `function`, arrow-function, named-export, or default-export forms.
- Type simple one-off props inline. Name a `Props` type when it is reused, exported, complex, or materially clearer.
- Use API-generated or API-returned types at component boundaries. Keep one-off UI refinements and conversions beside their owner.
- Name props and converted data after their domain/API role, preserving traceability to the original contract.
- Preserve domain value types for selections. Do not widen enums, unions, booleans, numbers, objects, or nullable values to `string` before a real boundary requires it.
- Avoid generic `common.tsx` buckets and aliases that only rename another type. Name files, values, and public types after their domain role.
- Put fallback and invariant checks in the lowest component that already renders that state. Do not extract helpers whose only purpose is hiding missing display data.

View File

@@ -4,13 +4,9 @@ Read this document when a change introduces Effects, navigation side effects, me
## Effects
- Keep render pure: do not read or write `ref.current` during render except for predictable null-guarded lazy initialization. Update interaction-owned refs in event handlers, synchronize external-system refs after commit, and use state or derivation for rendered values.
- Use Effects only to synchronize with a named external system such as a browser API, subscription, timer, analytics integration, non-React widget, or imperative DOM API.
- Do not use Effects to transform render state, handle user actions, copy query data, reset state from props, or fetch data owned by framework APIs or TanStack Query.
- Initialize query-backed form sessions after their defaults are available instead of copying data through Effects. Use a stable semantic identity key when the represented identity changes; use the intended surface lifecycle for per-session reset.
- When external synchronization participates in a feature state graph, prefer a small headless runtime controller that reads and writes focused atoms. Do not hide storage, polling, invalidation, and workflow refs in a giant hook that returns a state-and-handler object for another component or provider to redistribute.
- React state is not an immediate mutex for async serializers, debounce pipelines, migrations, or latest-wins queues because state updates do not synchronously change an in-flight closure. Keep renderable progress in state, but use owner-local refs for immediate locks, active operation identities, and queued latest snapshots; update those refs before awaiting work or scheduling the corresponding state update.
- A state value and ref may describe one workflow only when their roles are explicit: state renders the external status, while the ref coordinates events between renders. Retire both at the same success, failure, cancellation, identity-change, and unmount boundaries so neither can resurrect stale work.
- Initialize query-backed forms with keyed remounts or surface-entry hydration instead of copying data through Effects.
## Navigation
@@ -21,7 +17,6 @@ Read this document when a change introduces Effects, navigation side effects, me
## Runtime Cost
- Move changing state to the smallest consumer before considering memoization. Stable parent content can be lifted and passed as children.
- When a broad state owner causes unrelated rendering, move the owner or expose a narrower atom/query selector before adding memoization.
- Avoid `memo`, `useMemo`, and `useCallback` unless identity or computation has a demonstrated consumer or measurable cost.
- Start independent remote work together and await it near the branch that consumes it. Avoid introducing request waterfalls.
- Load heavy optional surfaces on demand when they sit behind a dialog, tab, command, or feature activation.

View File

@@ -2,87 +2,37 @@
Read this document when a change involves Jotai, form drafts, route identity, shared client state, or local persistence.
## Classify State On Two Axes
First identify the owner or source of truth: one component, a parent boundary, URL/framework APIs, TanStack Query, or the feature workflow. Then identify the graph role independently: primitive input, query/mutation node, derived business fact, or workflow command. Finally mark whether any node needs per-instance isolation or reset; this is a lifecycle requirement, not a node type.
A value does not enter the feature state graph solely because it drives an owner-local query or command. It should enter when it participates in shared workflow state: another owner or graph node consumes it, several consumers need one external-source bridge, it feeds reusable business derivation or coordinated writes, or its persistence/isolation must outlive the current owner. Entering the graph does not transfer ownership from URL APIs or Query cache to Jotai.
Once a value enters the graph, represent it in a feature-local Jotai state file by default. Do not leave graph inputs and async nodes inside a component or large custom hook while exporting only the final result through props or Context; that hides the dependency graph instead of modeling it.
Keep values out of the graph when they only affect one component's presentation, are read only at form submission, or are one-off render computations without domain meaning.
## Choose The Owner
- Keep synchronous state local when one component owns it: dialog and menu state, confirmations, field drafts, and local selections usually belong to the component or DOM.
- Use feature-scoped Jotai when siblings need one source of truth, values drive other atoms, a parent input establishes its own state graph, or a scoped workflow must preserve state across hidden or unmounted steps.
- Use feature-scoped Jotai when siblings need one source of truth, values drive other atoms, or a scoped workflow must preserve state across hidden or unmounted steps.
- Keep server and cache state in TanStack Query. Use existing feature stores for complex, high-frequency interaction state such as workflow canvas drag, resize, and runtime panels.
- Use feature-owned storage only for low-frequency client preferences, dismissed notices, and UI defaults. Live application state does not belong in local storage.
- Use Context for an existing authoritative product boundary or stable dependency injection. Do not introduce Context merely to translate a hook result, query observer, or prop fan-out into another large value object.
## Forms And Sessions
## Forms
- Follow the [form contract] for field controlledness and the [overlay contract] for Root/content mounting and state lifetime. These primitive contracts also apply when reviewing a consumer.
- Choose the Web draft owner from the required lifetime. Keep mounted-session drafts in the content owner; start with the lowest shared React owner when another component needs the draft or it must survive unmounting. Use feature-scoped atoms only when their coordination or persistence contract is needed.
- For query-backed defaults, establish the form session after the required defaults are available. `defaultValue` initializes the current mount; a stable semantic identity key may create a fresh snapshot when the represented identity changes. Do not use a generated key as a routine reset command.
- Prefer uncontrolled Dify UI form and field controls when values are only read at submit time. Initialize query-backed defaults with `defaultValue` and keyed remounts.
- Promote form values to atoms only when another owner reacts to in-progress values, the draft must survive scoped unmounting, or several workflow steps edit the same draft.
- Keep validation, source priority, fallback behavior, dirty checks, and payload assembly in the workflow that owns submission.
- Derive booleans, disabled flags, default tabs, and loading labels from current state. Do not mirror one value into competing prop, default, and local sources; controlled state alone is not a competing-source defect.
- Do not use local state to fake server data or generated contract fields, or connect a feature mock shell to an unrelated API before its actual contract is confirmed.
## Route And URL State
- Treat `useParams`, route arguments, and `nuqs` as the owners of URL identity and updates.
- A single consumer should read the owner hook directly. When several consumers, query atoms, or shared derived atoms require route identity, first match the bridge lifetime to the route or surface instance. An unscoped primitive is safe only when the owning store cannot contain two live instances of that bridge; otherwise scope the route primitive by instance. Keep URL writes in route and query-state APIs.
- A route, framework, or permission bridge that must follow later owner changes should use `dangerouslyForceHydrate: true` at that controlled boundary so descendants observe one synchronous authoritative snapshot. This is the canonical exception to ordinary render-phase write guidance when the hydration tuples contain only those authoritative external inputs and the target store or scope belongs to the current instance. The resulting React warning alone is not an ownership problem, but it must not be used to dismiss cross-instance sharing.
- A scoped workflow input is different: initialize it once, key the scope by semantic identity when switching entities should reset it, and do not force later parent refreshes into an in-progress session. Never extend the bridge exception to feature-owned primitives, drafts, query or mutation atoms, or edit-session snapshots.
- Hydrate a primitive atom at the route or surface boundary only when query atoms or shared derived atoms require route identity. Keep URL writes in route and query-state APIs.
- Within one route-owned feature, choose one route-identity source. Do not hydrate route identity into atoms while also threading the same ID through multiple component layers.
- A route parameter may cross the route-to-feature entry edge once. After a route bridge exists, queries, facts, commands, and descendant surfaces must read that bridge instead of accepting the same ID as props.
- Put shareable filters, tabs, selected panels, pagination, and search state in the URL. Keep one-shot navigation signals and transient UI state out of persistent subscriptions.
- Put shareable filters, tabs, pagination, and search state in the URL. Keep one-shot navigation signals and transient UI state out of persistent subscriptions.
## Build A Clean Jotai Graph
## Jotai And Query
- Keep one feature-local state file or folder ordered by dependency: primitives and boundary inputs, query atoms, query-data selectors, business facts, commands, mutation atoms when justified, submission orchestration, then provider exports.
- Keep graph flow one-way: primitive inputs -> queries -> named facts -> commands. Do not let components maintain a second writable entry for the same fact.
- Create a derived atom only when its value is reused, consumed by another atom, or deserves a stable business name such as `rows`, `showEmpty`, or `canSubmit`. Leave one-off presentation assembly in the component.
- Keep internal graph nodes unexported. Export only atoms that a component, provider boundary, or another state module must read or write.
- A Jotai-backed feature may keep one feature-local state module ordered by dependency: types and constants, primitives, query atoms, query-data derivations, business facts, commands, mutations, submission orchestration, and provider exports.
- Use `atomWithQuery` or `atomWithMutation` for async work driven by atom state. Do not hand-roll loading, error, or in-flight state for atom-orchestrated work.
- Use `selectAtom` or another field-specific derived atom for query results. `jotai-tanstack-query` does not provide TanStack Query tracked properties, so reading a whole query atom subscribes to the entire observer result. Read the whole result only when the consumer truly needs observer methods or a coordinated group of fields.
- Name derived atoms as business facts and write atoms as user or workflow commands. Commands should express actions such as selecting a source or submitting a wizard, not merely rename setter callbacks.
- Treat query keys, observer methods, cache invalidation, retries, and refresh composition as graph internals. Export a named command such as `refreshDocument`, not the query key plus a raw `refetch` callback.
- A headless runtime controller may synchronize atoms with storage, timers, subscriptions, or other external systems. It should read and write focused graph nodes and render no UI; it must not return a large object for a parent to redistribute.
## Scope, Isolation, And Reset
- Use field-specific derived atoms for query results. `jotai-tanstack-query` does not provide TanStack Query tracked properties, so reading a whole query atom subscribes to the entire observer result.
- Leave query and mutation atoms unscoped so they retain the shared QueryClient cache. Scope resettable primitives and hydration tuples; scope a derived atom only when all dependencies should be private to the surface.
- `jotai-scope` recursively scopes the dependencies of an explicitly scoped derived or write atom. Review that transitive dependency closure, not only the provider's atom list: scoping a fact or command that reaches a query atom can also privatize route inputs, `queryClientAtom`, and other cache dependencies. Keep query-facing facts and commands outside that scope, or redesign the command to receive the event snapshot it needs instead of reading the query graph.
- Do not add a derived fact or write command to the scoped list merely because it reads or writes scoped primitives. When consumed inside the scope, an unlisted fact or command can resolve those selected primitive dependencies from the current scope while retaining access to unscoped parent atoms; listing it instead recursively privatizes its dependency closure.
- Prefer `ScopeProvider` when an instance must isolate selected workflow atoms while retaining access to atoms in the parent application store. A nested `Provider` or `createStore` boundary cuts interoperability with every parent atom, not only `queryClientAtom`; injecting the current QueryClient repairs cache access but does not restore other global state. Use a fresh store only when full-store isolation is intentional, then inject every required external dependency explicitly.
- Use parent-store interoperation as the boundary test: if descendants must observe application auth, workspace, permissions, cache, or any other parent atom, a fresh `Provider` is not an instance-scope mechanism. Choose it only when the surface intentionally owns a complete store and must not observe later parent-atom updates.
- For an intentionally unscoped route or parent-input bridge, call `useHydrateAtoms` in the boundary component that returns `ScopeProvider`; hooks read the nearest provider available where that component itself renders. When the bridge requires per-instance isolation, initialize its primitives through the `ScopeProvider` tuples instead. If later authoritative owner changes must follow, render a dedicated hydration bridge beneath that provider so `useHydrateAtoms` resolves the scoped primitives.
- Use `atomWithLazy` or another non-null lazy primitive for required values injected by a scope provider; fail when the boundary forgot to provide them instead of inventing an empty ID.
- Maintain the scoped primitive list explicitly. Use a semantic `key` when switching identity should create a fresh session; do not use forced hydration for an edit snapshot that must remain stable while an outer query refreshes.
- Scope exists for per-instance isolation and natural reset, not as a general module boundary. A state file may be warranted even when its atoms remain unscoped.
- Use non-null lazy primitives for values always hydrated by a scope provider. Name derived atoms as business facts and write atoms as user or workflow commands.
- Keep independent dialog lifecycles separate. A scoped open-state atom is acceptable only when composed sibling surfaces would otherwise pass confusing lifecycle props through unrelated owners.
## Graph Review Checklist
Before implementation and again after the final slice, draw the actual graph in dependency order and verify:
- every route, URL, parent, and persisted input has one bridge;
- every query driven by graph input is represented in the graph and remains unscoped;
- components read field selectors or named facts rather than complete query results;
- write atoms and exported commands describe user or workflow intent;
- scoped atoms are limited to primitives and snapshots with a documented reset boundary;
- the scope still observes required parent/global atom updates, including the application's shared QueryClient;
- two simultaneously mounted route or surface instances with different identities or URL state do not overwrite each other's bridged primitives or scoped workflow state, and a semantic identity change resets the intended state;
- no equivalent value remains available through both atoms and descendant props;
- every exported atom has a real component, boundary, or state-module consumer.
## Persistence
- Use feature-owned storage modules built on `createLocalStorageState`; callers should not scatter direct storage access or raw keys.
- Persist high-frequency interaction state only on commit or after updates settle.
- Do not add ad hoc global event listeners for shared state. Centralize subscriptions through the owning atom, store, or subscription hook.
[form contract]: ../../../../packages/dify-ui/docs/forms.md
[overlay contract]: ../../../../packages/dify-ui/docs/overlays.md

View File

@@ -1,15 +0,0 @@
{
"hooks": {
"PreToolUse": [
{
"matcher": "Bash",
"hooks": [
{
"type": "command",
"command": "npx -y block-no-verify@1.1.1"
}
]
}
]
}
}

View File

@@ -12,7 +12,7 @@
"features": {
"ghcr.io/devcontainers/features/node:1": {
"nodeGypDependencies": true,
"version": "24.20.0"
"version": "lts"
},
"ghcr.io/devcontainers-extra/features/npm-package:1": {
"package": "typescript",
@@ -46,4 +46,4 @@
// Configure tool-specific properties.
// "customizations": {},
// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
}
}

View File

@@ -8,7 +8,6 @@
**/*.pyc
**/.mypy_cache
**/.ruff_cache
knowledge-fs/
.git
.github
*.md

View File

@@ -37,7 +37,3 @@ indent_size = 2
[package.json]
indent_style = space
indent_size = 2
# Oxlint writes its generated suppression baseline without a final newline.
[oxlint-suppressions.json]
insert_final_newline = false

13
.gemini/config.yaml Normal file
View File

@@ -0,0 +1,13 @@
have_fun: false
memory_config:
disabled: false
code_review:
disable: true
comment_severity_threshold: MEDIUM
max_review_comments: -1
pull_request_opened:
help: false
summary: false
code_review: false
include_drafts: false
ignore_patterns: []

28
.github/CODEOWNERS vendored
View File

@@ -26,34 +26,14 @@
/cli/ @GareArc
/.github/workflows/cli-tests.yml @GareArc
# Dify Agent
/dify-agent/ @BeautyyuYanli
# E2E
/e2e/ @lyzno1
/.github/workflows/web-e2e.yml @lyzno1
# Backend (default owner, more specific rules below will override)
/api/ @QuantumGhost
# Backend - Tests (default owners, domain-specific rules below will override)
# Backend - Tests
/api/tests/ @laipz8200 @QuantumGhost
/api/tests/*/core/rag/ @JohnJyong
/api/tests/*/services/rag_pipeline/ @JohnJyong
/api/tests/*/services/dataset_service/ @JohnJyong
/api/tests/*/controllers/console/datasets/ @JohnJyong
/api/tests/*/controllers/service_api/dataset/ @JohnJyong
/api/tests/*/vdb/ @JohnJyong
/api/tests/*/core/plugin/ @WH-2099
/api/tests/*/services/plugin/ @WH-2099
/api/tests/*/controllers/inner_api/plugin/ @WH-2099
/api/tests/*/plugin/ @WH-2099
/api/tests/*/controllers/trigger/ @CourTeous33
/api/tests/*/core/trigger/ @CourTeous33
/api/tests/*/trigger/ @CourTeous33
/api/tests/*/controllers/console/billing/ @hj24 @zyssyz123
/api/tests/*/services/enterprise/ @GareArc
/api/tests/*/migrations/ @snakevash @laipz8200 @MRZHUH
# Backend - RAG (Retrieval Augmented Generation)
/api/core/rag/ @JohnJyong
@@ -265,7 +245,7 @@
# Frontend - Billing and Education
/web/app/components/billing/ @iamjoel @zxhlyh
/web/app/education/ @iamjoel @zxhlyh
/web/app/education-apply/ @iamjoel @zxhlyh
# Frontend - Workspace
/web/app/components/header/account-dropdown/workplace-selector/ @iamjoel @zxhlyh
@@ -287,9 +267,5 @@
/web/app/auth/refresh/route.ts @iamjoel @lyzno1
/web/service/server.ts @iamjoel @lyzno1
# Frontend - Browser Mode Tests
/web/app/**/*.browser.spec.ts @lyzno1
/web/app/**/*.browser.spec.tsx @lyzno1
# Docker
/docker/* @laipz8200

View File

@@ -1,7 +1,7 @@
name: '🕷️ Bug report'
description: Report errors or unexpected behavior
labels:
- 🐞 bug
- bug
body:
- type: checkboxes
attributes:

View File

@@ -1,7 +1,7 @@
name: '⭐ Feature or enhancement request'
description: Propose something new.
labels:
- 💪 enhancement
- enhancement
body:
- type: checkboxes
attributes:

View File

@@ -4,13 +4,13 @@ description: Set up Node.js, Vite+, pnpm, and web dependencies
runs:
using: composite
steps:
- name: Setup pnpm and Node.js
uses: pnpm/setup@703c52620218391530e48b9e8870d5c0082e1b9b # v2.1.0
- name: Setup pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
with:
install: false
run_install: false
- name: Setup Vite+
uses: voidzero-dev/setup-vp@1b32467adbe183473499fd9d5d372c3ed9641754 # v1.18.0
uses: voidzero-dev/setup-vp@143f5f385f39b1b753ffed1a01ad443811855c8b # v1.16.1
with:
node-manager: false
node-version-file: .nvmrc
cache: true
run-install: true

View File

@@ -1,15 +1,6 @@
version: 2
updates:
- package-ecosystem: 'npm'
directory: '/knowledge-fs'
open-pull-requests-limit: 10
schedule:
interval: 'weekly'
groups:
knowledge-fs-dependencies:
patterns:
- '*'
- package-ecosystem: 'uv'
directory: '/api'
open-pull-requests-limit: 10

1
.github/labeler.yml vendored
View File

@@ -6,6 +6,7 @@ web:
- 'package.json'
- 'pnpm-lock.yaml'
- 'pnpm-workspace.yaml'
- '.nvmrc'
e2e:
- changed-files:

View File

@@ -4,8 +4,6 @@ extends: default
rules:
brackets:
max-spaces-inside: 1
comments:
min-spaces-from-content: 1
comments-indentation: disable
document-start: disable
line-length: disable

View File

@@ -21,4 +21,4 @@
- [ ] I understand that this PR may be closed in case there was no previous discussion or issues. (This doesn't apply to typos!)
- [ ] I've added a test for each change that was introduced, and I tried as much as possible to make a single atomic change.
- [ ] I've updated the documentation accordingly.
- [ ] I ran `make lint && make type-check` (backend) and `vp staged` (frontend) to appease the lint gods
- [ ] I ran `make lint && make type-check` (backend) and `cd web && pnpm exec vp staged` (frontend) to appease the lint gods

View File

@@ -36,7 +36,7 @@ concurrency:
jobs:
accessibility:
name: WCAG Level ${{ inputs.level == 'a' && 'A' || 'AA' }} · ${{ inputs.page }}
runs-on: ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
timeout-minutes: 120
defaults:
run:
@@ -52,7 +52,7 @@ jobs:
uses: ./.github/actions/setup-web
- name: Setup UV and Python
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true
python-version: '3.12'

View File

@@ -35,7 +35,7 @@ jobs:
persist-credentials: false
- name: Setup UV and Python
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true
python-version: ${{ matrix.python-version }}
@@ -71,10 +71,10 @@ jobs:
retention-days: 1
api-integration:
name: API Integration Tests (${{ matrix.python-version }}, shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
runs-on: depot-ubuntu-24.04-4
name: API Integration Tests
runs-on: depot-ubuntu-24.04
env:
COVERAGE_FILE: coverage-integration-${{ matrix.python-version }}-${{ matrix.shardIndex }}
COVERAGE_FILE: coverage-integration
STORAGE_TYPE: opendal
OPENDAL_SCHEME: fs
OPENDAL_FS_ROOT: /tmp/dify-storage
@@ -82,17 +82,9 @@ jobs:
run:
shell: bash
strategy:
fail-fast: false
matrix:
python-version:
- '3.12'
shardIndex:
- 1
- 2
- 3
- 4
shardTotal:
- 4
steps:
- name: Checkout code
@@ -102,7 +94,7 @@ jobs:
persist-credentials: false
- name: Setup UV and Python
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true
python-version: ${{ matrix.python-version }}
@@ -120,8 +112,6 @@ jobs:
-p no:benchmark \
--start-middleware \
-n auto \
--shard-index "${{ matrix.shardIndex }}" \
--shard-total "${{ matrix.shardTotal }}" \
--timeout "${PYTEST_TIMEOUT:-180}" \
api/tests/integration_tests/workflow \
api/tests/integration_tests/tools \
@@ -130,9 +120,8 @@ jobs:
- name: Upload integration coverage data
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: api-coverage-integration-${{ matrix.python-version }}-${{ matrix.shardIndex }}
path: ${{ env.COVERAGE_FILE }}
if-no-files-found: error
name: api-coverage-integration
path: coverage-integration
retention-days: 1
api-coverage:
@@ -156,7 +145,7 @@ jobs:
persist-credentials: false
- name: Setup UV and Python
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true
python-version: '3.12'
@@ -182,15 +171,12 @@ jobs:
echo "" >> "$GITHUB_STEP_SUMMARY"
unit_coverage="$(find coverage-data -type f -name coverage-unit -print -quit)"
mapfile -t integration_coverage < <(find coverage-data -type f -name 'coverage-integration-*' -print | sort)
integration_coverage="$(find coverage-data -type f -name coverage-integration -print -quit)"
: "${unit_coverage:?coverage-unit artifact not found}"
if [[ "${#integration_coverage[@]}" -ne 4 ]]; then
echo "expected 4 integration coverage artifacts, found ${#integration_coverage[@]}" >&2
exit 1
fi
: "${integration_coverage:?coverage-integration artifact not found}"
report_file="$(mktemp)"
uv run --project api coverage combine "$unit_coverage" "${integration_coverage[@]}"
uv run --project api coverage combine "$unit_coverage" "$integration_coverage"
uv run --project api coverage report --show-missing | tee "$report_file"
echo "Summary: \`$(tail -n 1 "$report_file")\`" >> "$GITHUB_STEP_SUMMARY"
{

View File

@@ -5,6 +5,8 @@ on:
merge_group:
branches: ['main']
types: [checks_requested]
push:
branches: ['main']
permissions:
contents: read
@@ -30,6 +32,28 @@ jobs:
docker/.env.example
docker/docker-compose-template.yaml
docker/docker-compose.yaml
- name: Check web inputs
if: github.event_name != 'merge_group'
id: web-changes
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
with:
files: |
web/**
cli/**
e2e/**
packages/**
sdks/nodejs-client/**
package.json
pnpm-lock.yaml
pnpm-workspace.yaml
.nvmrc
vite.config.ts
lint.config.ts
eslint.config.mjs
oxlint-suppressions.json
eslint-suppressions.json
.vscode/**
.github/**
- name: Check api inputs
if: github.event_name != 'merge_group'
id: api-changes
@@ -58,13 +82,13 @@ jobs:
dify-agent/**/*.py
dify-agent/pyproject.toml
dify-agent/uv.lock
- if: github.event_name != 'merge_group' && (steps.docker-compose-changes.outputs.any_changed == 'true' || steps.api-changes.outputs.any_changed == 'true' || steps.frontend-contract-changes.outputs.any_changed == 'true' || steps.dify-agent-changes.outputs.any_changed == 'true')
- if: github.event_name != 'merge_group'
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: '3.11'
- if: github.event_name != 'merge_group' && (steps.api-changes.outputs.any_changed == 'true' || steps.frontend-contract-changes.outputs.any_changed == 'true' || steps.dify-agent-changes.outputs.any_changed == 'true')
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
- if: github.event_name != 'merge_group'
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
- name: Generate Docker Compose
if: github.event_name != 'merge_group' && steps.docker-compose-changes.outputs.any_changed == 'true'
@@ -94,6 +118,12 @@ jobs:
# Format code
uv run ruff format .
- name: count migration progress
if: github.event_name != 'merge_group' && steps.api-changes.outputs.any_changed == 'true'
run: |
cd api
./cnt_base.sh
- name: ast-grep
if: github.event_name != 'merge_group' && steps.api-changes.outputs.any_changed == 'true'
run: |
@@ -125,7 +155,7 @@ jobs:
find . -name "*.py.bak" -type f -delete
- name: Setup web environment
if: github.event_name != 'merge_group' && steps.frontend-contract-changes.outputs.any_changed == 'true'
if: github.event_name != 'merge_group'
uses: ./.github/actions/setup-web
- name: Generate API docs
@@ -138,5 +168,13 @@ jobs:
if: github.event_name != 'merge_group' && steps.frontend-contract-changes.outputs.any_changed == 'true'
run: pnpm --dir packages/contracts gen-api-contract
- name: ESLint fallback autofix
if: github.event_name != 'merge_group' && steps.web-changes.outputs.any_changed == 'true'
run: pnpm lint:eslint:fix --quiet || true
- name: Vite+ static autofix
if: github.event_name != 'merge_group' && (steps.web-changes.outputs.any_changed == 'true' || steps.frontend-contract-changes.outputs.any_changed == 'true')
run: vp check --fix || true
- if: github.event_name != 'merge_group'
uses: autofix-ci/action@c5b2d67aa2274e7b5a18224e8171550871fc7e4a # v1.3.4

View File

@@ -103,7 +103,7 @@ jobs:
password: ${{ env.DOCKERHUB_TOKEN }}
- name: Set up Depot CLI
uses: depot/setup-action@91bc8495a33ebfc504ffc89e5674379ccf23c29c # v1.7.2
uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.7.1
- name: Extract metadata for Docker
id: meta
@@ -160,7 +160,7 @@ jobs:
file: 'docker/Dockerfile'
steps:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Validate Docker image
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
@@ -229,32 +229,3 @@ jobs:
IMAGE_VERSION: ${{ steps.meta.outputs.version }}
run: |
docker buildx imagetools inspect "$IMAGE_NAME:$IMAGE_VERSION"
sync-e2b-template:
needs: create-manifest
runs-on: ubuntu-24.04
if: github.repository == 'langgenius/dify' && startsWith(github.ref, 'refs/tags/')
strategy:
matrix:
include:
- project: dev
api_key_secret: E2B_API_KEY_DEV
- project: prod
api_key_secret: E2B_API_KEY_PROD
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
- name: Login to Docker Hub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
username: ${{ env.DOCKERHUB_USER }}
password: ${{ env.DOCKERHUB_TOKEN }}
- name: Sync E2B Template (${{ matrix.project }})
env:
E2B_API_KEY: ${{ secrets[matrix.api_key_secret] }}
run: ./dify-agent-runtime/docker/sync-e2b-template.sh

View File

@@ -87,9 +87,10 @@ jobs:
with:
bun-version: latest
- uses: pnpm/setup@703c52620218391530e48b9e8870d5c0082e1b9b # v2.1.0
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
with:
install: false
package_json_field: packageManager
run_install: false
- name: Install CLI dependencies
working-directory: cli
@@ -111,7 +112,7 @@ jobs:
# ════════════════════════════════════════════════════════════════════════════
suite-framework-output-error:
name: 'Suite: framework + output + error-handling'
if: ${{ inputs.suite_framework_output_error }}
if: ${{ inputs.suite_framework_output_error != 'false' }}
needs: provision
runs-on: ubuntu-latest
timeout-minutes: 20
@@ -128,8 +129,10 @@ jobs:
- uses: ./.github/actions/setup-web
- uses: oven-sh/setup-bun@v2
with:
bun-version: latest
with: { bun-version: latest }
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
with: { package_json_field: packageManager, run_install: false }
- run: pnpm install --frozen-lockfile
- run: pnpm tree:gen
- name: Run framework + output + error-handling
@@ -156,7 +159,7 @@ jobs:
# ════════════════════════════════════════════════════════════════════════════
suite-discovery:
name: 'Suite: discovery'
if: ${{ inputs.suite_discovery }}
if: ${{ inputs.suite_discovery != 'false' }}
needs: provision
runs-on: ubuntu-latest
timeout-minutes: 20
@@ -173,8 +176,10 @@ jobs:
- uses: ./.github/actions/setup-web
- uses: oven-sh/setup-bun@v2
with:
bun-version: latest
with: { bun-version: latest }
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
with: { package_json_field: packageManager, run_install: false }
- run: pnpm install --frozen-lockfile
- run: pnpm tree:gen
- name: Run discovery suite
@@ -202,7 +207,7 @@ jobs:
# ════════════════════════════════════════════════════════════════════════════
suite-run:
name: 'Suite: run / ${{ matrix.name }}'
if: ${{ inputs.suite_run }}
if: ${{ inputs.suite_run != 'false' }}
needs: provision
runs-on: ubuntu-latest
timeout-minutes: 20
@@ -234,8 +239,10 @@ jobs:
- uses: ./.github/actions/setup-web
- uses: oven-sh/setup-bun@v2
with:
bun-version: latest
with: { bun-version: latest }
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
with: { package_json_field: packageManager, run_install: false }
- run: pnpm install --frozen-lockfile
- run: pnpm tree:gen
- name: 'Run run/${{ matrix.name }}'
@@ -277,7 +284,7 @@ jobs:
# ════════════════════════════════════════════════════════════════════════════
suite-auth-safe:
name: 'Suite: auth (login / status / whoami)'
if: ${{ inputs.suite_auth }}
if: ${{ inputs.suite_auth != 'false' }}
needs: provision
runs-on: ubuntu-latest
timeout-minutes: 15
@@ -294,8 +301,10 @@ jobs:
- uses: ./.github/actions/setup-web
- uses: oven-sh/setup-bun@v2
with:
bun-version: latest
with: { bun-version: latest }
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
with: { package_json_field: packageManager, run_install: false }
- run: pnpm install --frozen-lockfile
- run: pnpm tree:gen
- name: Run auth/login + status + whoami
@@ -324,7 +333,7 @@ jobs:
suite-last:
name: 'Suite: auth-use + devices + logout + agent (last, serial)'
# Runs when auth is selected; also runs after all parallel jobs finish
if: ${{ inputs.suite_auth || inputs.suite_agent }}
if: ${{ inputs.suite_auth != 'false' || inputs.suite_agent != 'false' }}
needs:
- provision
- suite-framework-output-error
@@ -348,8 +357,10 @@ jobs:
- uses: ./.github/actions/setup-web
- uses: oven-sh/setup-bun@v2
with:
bun-version: latest
with: { bun-version: latest }
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
with: { package_json_field: packageManager, run_install: false }
- run: pnpm install --frozen-lockfile
- run: pnpm tree:gen
- name: Run use / devices / logout / agent (serial)

View File

@@ -2,11 +2,10 @@ name: CLI Edge Publish
on:
push:
branches: [main, deploy/konwledge]
branches: [main]
paths:
- 'cli/**'
- 'packages/contracts/generated/api/openapi/**'
- '.github/workflows/cli-edge.yml'
workflow_dispatch:
concurrency:

View File

@@ -7,22 +7,12 @@ on:
description: Dify release tag to attach difyctl assets to (blank = latest stable)
required: false
type: string
dry_run:
description: Build and checksum only — skip asset upload and stale-asset prune
required: false
type: boolean
default: false
workflow_call:
inputs:
release_tag:
description: Dify release tag to attach difyctl assets to (blank = latest stable)
required: false
type: string
dry_run:
description: Build and checksum only — skip asset upload and stale-asset prune
required: false
type: boolean
default: false
release:
types: [released]
@@ -49,8 +39,11 @@ jobs:
with:
persist-credentials: false
- name: Export manifest to env
run: node scripts/release-naming.mjs github-env >> "$GITHUB_ENV"
- name: Validate manifest
run: node scripts/release-naming.mjs validate
run: scripts/release-validate-manifest.sh
- name: Resolve target Dify release
id: resolve
@@ -82,6 +75,15 @@ jobs:
DIFY_TAG: ${{ steps.resolve.outputs.dify_tag }}
run: node scripts/release-naming.mjs compat-check "$DIFY_TAG"
- name: Reject duplicate difyctl version
env:
GH_TOKEN: ${{ github.token }}
run: |
if gh api "repos/${GITHUB_REPOSITORY}/git/ref/tags/${difyctlTag}" >/dev/null 2>&1; then
echo "::error::difyctl ${version} already released (tag ${difyctlTag} exists); bump cli/package.json version"
exit 1
fi
release:
name: build + attach standalone binaries (all targets)
needs: validate
@@ -118,7 +120,6 @@ jobs:
- name: Compile standalone binaries (all targets)
run: |
CLI_VERSION="$version" \
DIFYCTL_COMMIT="$(git rev-parse HEAD)" \
DIFYCTL_BUILD_DATE="$(git log -1 --format=%cI HEAD)" \
pnpm build:bin
@@ -127,7 +128,6 @@ jobs:
run: scripts/release-write-checksums.sh
- name: Attach difyctl assets to Dify release
if: ${{ !inputs.dry_run }}
env:
GH_TOKEN: ${{ github.token }}
run: |
@@ -135,7 +135,6 @@ jobs:
--repo "$GITHUB_REPOSITORY" --clobber
- name: Prune stale difyctl assets
if: ${{ !inputs.dry_run }}
env:
GH_TOKEN: ${{ github.token }}
run: |
@@ -150,3 +149,18 @@ jobs:
--repo "$GITHUB_REPOSITORY" --yes
fi
done
- name: Create provenance tag
env:
GH_TOKEN: ${{ github.token }}
run: |
ref="refs/tags/${difyctlTag}"
sha="$(git rev-parse HEAD)"
status="$(gh api -X POST "repos/${GITHUB_REPOSITORY}/git/refs" \
-f ref="$ref" -f sha="$sha" --silent --include 2>/dev/null \
| awk 'NR==1 {print $2; exit}' || true)"
case "$status" in
201) echo "::notice::created ${ref}" ;;
422) echo "::notice::tag ${ref} already exists; skipping (immutable)" ;;
*) echo "::error::provenance tag ${ref} not created (HTTP ${status:-unknown})"; exit 1 ;;
esac

View File

@@ -44,7 +44,7 @@ jobs:
- name: Validate release manifest
if: matrix.os == 'depot-ubuntu-24.04'
run: node scripts/release-naming.mjs validate
run: scripts/release-validate-manifest.sh
- name: CI pipeline (tree, coverage, build)
run: pnpm run ci

View File

@@ -19,7 +19,7 @@ jobs:
persist-credentials: false
- name: Setup UV and Python
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true
python-version: '3.12'
@@ -40,7 +40,7 @@ jobs:
cp envs/middleware.env.example middleware.env
- name: Set up Middlewares
uses: hoverkraft-tech/compose-action@ee6af68587292d72db67743171809c19787df4c9 # v3.1.0
uses: hoverkraft-tech/compose-action@11beaa1c2dae4e8ed7b1665aa074723b6cecb0e4 # v3.0.0
with:
compose-file: |
docker/docker-compose.middleware.yaml
@@ -69,7 +69,7 @@ jobs:
persist-credentials: false
- name: Setup UV and Python
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true
python-version: '3.12'
@@ -94,7 +94,7 @@ jobs:
sed -i 's/DB_USERNAME=postgres/DB_USERNAME=mysql/' middleware.env
- name: Set up Middlewares
uses: hoverkraft-tech/compose-action@ee6af68587292d72db67743171809c19787df4c9 # v3.1.0
uses: hoverkraft-tech/compose-action@11beaa1c2dae4e8ed7b1665aa074723b6cecb0e4 # v3.0.0
with:
compose-file: |
docker/docker-compose.middleware.yaml

View File

@@ -2,70 +2,23 @@ name: Deploy Dev
on:
workflow_run:
workflows: ['KnowledgeFS CI']
workflows: ['Build and Push API & Web']
branches:
- 'deploy/konwledge'
- 'deploy/dev'
types:
- completed
permissions: {}
jobs:
deploy:
runs-on: depot-ubuntu-24.04
permissions:
actions: read
contents: read
if: |
github.event.workflow_run.conclusion == 'success' &&
github.event.workflow_run.head_branch == 'deploy/konwledge'
github.event.workflow_run.head_branch == 'deploy/dev'
steps:
- name: Wait for API and Web image build
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
timeout-minutes: 35
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const workflowId = "build-push.yml";
const headBranch = context.payload.workflow_run.head_branch;
const headSha = context.payload.workflow_run.head_sha;
const deadline = Date.now() + 30 * 60 * 1000;
const pollIntervalMs = 15 * 1000;
while (Date.now() < deadline) {
const { data } = await github.rest.actions.listWorkflowRuns({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: workflowId,
branch: headBranch,
event: "push",
head_sha: headSha,
per_page: 10,
});
const run = data.workflow_runs[0];
if (!run) {
core.info(`Waiting for ${workflowId} to start for ${headSha}.`);
} else if (run.status !== "completed") {
core.info(`Waiting for ${run.html_url}; current status is ${run.status}.`);
} else if (run.conclusion !== "success") {
throw new Error(
`${workflowId} did not succeed for ${headSha}: ${run.conclusion} (${run.html_url})`,
);
} else {
core.info(`Both image workflows succeeded for ${headSha}: ${run.html_url}`);
return;
}
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
}
throw new Error(`Timed out waiting for ${workflowId} to succeed for ${headSha}.`);
- name: Deploy to server
uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5
with:
host: ${{ secrets.SSH_NEW_RAG_HOST }}
host: ${{ secrets.SSH_HOST }}
username: ${{ secrets.SSH_USER }}
key: ${{ secrets.SSH_PRIVATE_KEY }}
script: |

View File

@@ -1,25 +0,0 @@
name: Deploy HITL IM Dev
on:
workflow_run:
workflows: ['Build and Push API & Web']
branches:
- 'build/feat/hitl-im-dev'
types:
- completed
jobs:
deploy:
runs-on: depot-ubuntu-24.04
if: |
github.event.workflow_run.conclusion == 'success' &&
github.event.workflow_run.head_branch == 'build/feat/hitl-im-dev'
steps:
- name: Deploy to server
uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5
with:
host: ${{ secrets.HITL_SSH_HOST }}
username: ${{ secrets.SSH_USER }}
key: ${{ secrets.SSH_PRIVATE_KEY }}
script: |
${{ vars.SSH_SCRIPT || secrets.SSH_SCRIPT }}

View File

@@ -68,47 +68,4 @@ jobs:
username: ${{ secrets.SSH_USER }}
key: ${{ secrets.SSH_PRIVATE_KEY }}
script: |
set -euo pipefail
deploy_script="$(mktemp)"
normalized_script="${deploy_script}.normalized"
cleanup() {
rm -f "$deploy_script" "$normalized_script"
}
trap cleanup EXIT
cat > "$deploy_script" <<'DIFY_KNOWLEDGE_DEPLOY_SCRIPT'
${{ vars.SSH_KNOWLEDGE_SCRIPT || secrets.SSH_KNOWLEDGE_SCRIPT }}
DIFY_KNOWLEDGE_DEPLOY_SCRIPT
if [[ ! -s "$deploy_script" ]]; then
echo "KnowledgeFS deploy script is empty." >&2
exit 1
fi
# GitHub variables and secrets can retain Windows CRLF line endings.
# Normalize them before Bash parses options such as `set -eu`.
tr -d '\r' < "$deploy_script" > "$normalized_script"
mv "$normalized_script" "$deploy_script"
bash -e -u -o pipefail "$deploy_script"
- name: Apply KnowledgeFS database migrations
uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5
with:
host: ${{ secrets.SSH_NEW_RAG_HOST }}
username: ${{ secrets.SSH_USER }}
key: ${{ secrets.SSH_PRIVATE_KEY }}
script: |
set -euo pipefail
mapfile -t knowledge_fs_containers < <(
docker ps \
--filter 'label=com.docker.compose.service=knowledge_fs' \
--format '{{.ID}}'
)
if [[ "${#knowledge_fs_containers[@]}" -ne 1 ]]; then
echo "Expected exactly one running KnowledgeFS container, found ${#knowledge_fs_containers[@]}." >&2
exit 1
fi
docker exec "${knowledge_fs_containers[0]}" node migrate.mjs
${{ vars.SSH_SCRIPT || secrets.SSH_SCRIPT }}

View File

@@ -1,47 +0,0 @@
name: Dify Agent Tests
on:
workflow_call:
permissions:
contents: read
concurrency:
group: dify-agent-tests-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
dify-agent-unit:
name: Dify Agent Unit Tests
runs-on: depot-ubuntu-24.04
defaults:
run:
shell: bash
working-directory: dify-agent
strategy:
matrix:
python-version:
- '3.12'
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Setup UV and Python
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
with:
enable-cache: true
python-version: ${{ matrix.python-version }}
cache-dependency-glob: dify-agent/uv.lock
- name: Check UV lockfile
run: uv lock --check
- name: Install dependencies
run: uv sync --extra server --dev
- name: Run unit tests
run: uv run pytest

View File

@@ -9,13 +9,6 @@ on:
- api/Dockerfile.dockerignore
- api/pyproject.toml
- api/uv.lock
- knowledge-fs/apps/api/**
- knowledge-fs/packages/**
- knowledge-fs/package.json
- knowledge-fs/pnpm-lock.yaml
- knowledge-fs/pnpm-workspace.yaml
- knowledge-fs/tsconfig.base.json
- knowledge-fs/turbo.json
- dify-agent/pyproject.toml
- dify-agent/README.md
- dify-agent/src/**
@@ -72,7 +65,7 @@ jobs:
file: 'docker/Dockerfile'
steps:
- name: Set up Depot CLI
uses: depot/setup-action@91bc8495a33ebfc504ffc89e5674379ccf23c29c # v1.7.2
uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.7.1
- name: Build Docker Image
uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.18.0
@@ -103,7 +96,7 @@ jobs:
file: 'docker/Dockerfile'
steps:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Build Docker Image
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0

View File

@@ -1,542 +0,0 @@
name: KnowledgeFS CI
on:
pull_request:
branches: ['main']
merge_group:
branches: ['main']
types: [checks_requested]
push:
branches: ['main', 'deploy/konwledge']
workflow_dispatch:
permissions:
contents: read
pull-requests: read
concurrency:
group: knowledge-fs-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
CI: true
DIFY_KNOWLEDGE_FS_API_IMAGE_NAME: >-
${{ vars.DIFY_KNOWLEDGE_FS_API_IMAGE_NAME || 'langgenius/dify-knowledge-fs-api' }}
jobs:
check-changes:
name: Check KnowledgeFS changes
runs-on: depot-ubuntu-24.04-4
outputs:
knowledge-fs: ${{ steps.changes.outputs.knowledge-fs }}
steps:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0
persist-credentials: false
- name: Detect KnowledgeFS changes
id: changes
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
with:
filters: |
knowledge-fs:
- 'knowledge-fs/**'
- 'knowledge-fs/packages/api/src/dify-capability-v2.ts'
- 'knowledge-fs/packages/api/src/knowledge-space-routes.ts'
- 'knowledge-fs/packages/api/src/upload-session-routes.ts'
- 'knowledge-fs/scripts/export-capability-v2-operations.mjs'
- 'knowledge-fs/scripts/export-openapi.mjs'
- 'api/dev/generate_knowledge_fs_contract.py'
- 'api/dev/knowledge_fs_product_contract.py'
- 'api/knowledge-fs-contract.lock.json'
- 'api/knowledge-fs-product-operation-gaps.json'
- 'api/knowledge-fs-product-operations.json'
- 'api/**/knowledge_fs/**'
- 'api/**/*knowledge_fs*'
- 'api/**/*knowledge-fs*'
- 'api/.env.example'
- 'api/app_factory.py'
- 'api/commands/__init__.py'
- 'api/controllers/console/__init__.py'
- 'api/controllers/console/workspace/rbac.py'
- 'api/controllers/service_api/__init__.py'
- 'api/core/agent/base_agent_runner.py'
- 'api/core/app/apps/agent_app/runtime_request_builder.py'
- 'api/core/rbac/entities.py'
- 'api/core/tools/__base/tool_runtime.py'
- 'api/core/tools/builtin_tool/_position.yaml'
- 'api/core/workflow/node_runtime.py'
- 'api/core/workflow/nodes/agent_v2/runtime_request_builder.py'
- 'api/extensions/ext_celery.py'
- 'api/extensions/ext_commands.py'
- 'api/models/__init__.py'
- 'api/services/account_service.py'
- 'api/services/agent_tool_inner_service.py'
- 'api/services/enterprise/rbac_service.py'
- 'api/services/entities/agent_tool_inner.py'
- 'api/services/knowledge_fs/**'
- 'api/services/knowledge_fs_capability.py'
- 'api/tests/unit_tests/dev/test_generate_knowledge_fs_contract.py'
- 'api/tests/unit_tests/controllers/console/workspace/test_rbac.py'
- 'api/tests/unit_tests/core/agent/test_base_agent_runner.py'
- 'api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py'
- 'api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py'
- 'api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_runtime.py'
- 'api/tests/unit_tests/core/workflow/test_node_runtime.py'
- 'api/tests/unit_tests/services/enterprise/test_rbac_service.py'
- 'api/tests/unit_tests/services/test_account_service.py'
- 'api/tests/unit_tests/services/test_agent_tool_inner_service.py'
- 'api/tests/unit_tests/services/test_knowledge_fs_capability.py'
- 'api/tests/unit_tests/services/test_knowledge_fs_product_operations.py'
- 'api/pyproject.toml'
- 'api/uv.lock'
- 'dify-agent/src/dify_agent/layers/dify_core_tools/client.py'
- 'dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_client.py'
- 'packages/contracts/generated/api/console/**'
- 'packages/contracts/generated/api/service/**'
- 'docker/.env.example'
- 'docker/README.md'
- 'docker/dify-env-sync.py'
- 'docker/dify-env-sync.sh'
- 'docker/docker-compose-template.yaml'
- 'docker/docker-compose.yaml'
- 'docker/envs/core-services/api.env.example'
- 'docker/envs/core-services/knowledge-fs.env.example'
- 'docker/envs/core-services/knowledge-fs-unstructured-service.defaults'
- 'docker/envs/core-services/knowledge-fs-unstructured.env.example'
- 'docker/knowledge-fs-unstructured-sandbox.compose.yaml'
- 'docker/generate_docker_compose'
- 'docs/design/knowledge-fs*'
- '.github/dependabot.yml'
- '.github/workflows/deploy-knowledge.yml'
- '.github/workflows/knowledge-fs-ci.yml'
build:
name: Build KnowledgeFS API production image
needs:
- check-changes
- quality
if: >-
(needs.check-changes.outputs.knowledge-fs == 'true' || github.event_name == 'workflow_dispatch') &&
needs.quality.result == 'success'
runs-on: depot-ubuntu-24.04-4
steps:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Login to Docker Hub
if: >-
github.event_name == 'workflow_dispatch' ||
(github.event_name == 'push' &&
(github.ref == 'refs/heads/main' || github.ref == 'refs/heads/deploy/konwledge'))
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Extract KnowledgeFS image metadata
id: meta
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
with:
images: ${{ env.DIFY_KNOWLEDGE_FS_API_IMAGE_NAME }}
tags: |
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
type=ref,event=branch
type=sha,format=long
- name: Build KnowledgeFS API image
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: ./knowledge-fs
file: ./knowledge-fs/apps/api/Dockerfile
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/amd64
push: >-
${{ github.event_name == 'workflow_dispatch' ||
(github.event_name == 'push' &&
(github.ref == 'refs/heads/main' || github.ref == 'refs/heads/deploy/konwledge')) }}
tags: ${{ steps.meta.outputs.tags }}
quality:
name: Run KnowledgeFS quality and contract gates
needs: check-changes
if: needs.check-changes.outputs.knowledge-fs == 'true' || github.event_name == 'workflow_dispatch'
runs-on: depot-ubuntu-24.04-4
defaults:
run:
shell: bash
working-directory: ./knowledge-fs
steps:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0
persist-credentials: false
- name: Setup pnpm
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
with:
package_json_file: knowledge-fs/package.json
run_install: false
- name: Setup Node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
cache: pnpm
cache-dependency-path: knowledge-fs/pnpm-lock.yaml
- name: Install KnowledgeFS dependencies
run: pnpm install --frozen-lockfile
- name: Install PDF metadata inspector
run: |
sudo apt-get update
sudo apt-get install --yes --no-install-recommends poppler-utils
pdfinfo -v
- name: Scan KnowledgeFS secrets
run: pnpm security:secrets
- name: Audit KnowledgeFS production dependencies
run: pnpm security:dependencies
- name: Run KnowledgeFS checks
run: pnpm check
- name: Build KnowledgeFS
run: pnpm build
- name: Lint KnowledgeFS
run: pnpm lint:backend
- name: Setup UV and Python
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
with:
enable-cache: true
python-version: '3.12'
cache-dependency-glob: |
api/uv.lock
dify-agent/uv.lock
- name: Verify Dify dependency lock
working-directory: .
run: uv lock --project api --check
- name: Install Dify contract dependencies
working-directory: .
run: uv sync --project api --locked --dev
- name: Test dedicated parser sandbox
working-directory: .
env:
PYTHONPATH: knowledge-fs/services/unstructured-sandbox
run: uv run --project api python -m unittest discover -s knowledge-fs/services/unstructured-sandbox/tests -t knowledge-fs/services/unstructured-sandbox -v
- name: Collect Dify KnowledgeFS gate targets
working-directory: .
run: |
set -euo pipefail
target_dir="${RUNNER_TEMP:?}/knowledge-fs-ci-targets"
mkdir -p "$target_dir"
production_targets=()
add_production_target() {
local path="$1"
if [[ ! -f "$path" ]]; then
echo "required Dify KnowledgeFS production target is missing: $path" >&2
exit 1
fi
production_targets+=("$path")
}
while IFS= read -r -d '' path; do
if [[ "$path" == *knowledge_fs* ]]; then
add_production_target "$path"
fi
done < <(
find api \
\( -path 'api/.venv' -o -path 'api/tests' -o -path 'api/storage' \) -prune \
-o -type f -name '*.py' -print0
)
production_touchpoints=(
api/app_factory.py
api/commands/__init__.py
api/controllers/console/__init__.py
api/controllers/console/datasets/datasets.py
api/controllers/console/workspace/rbac.py
api/controllers/service_api/__init__.py
api/core/agent/base_agent_runner.py
api/core/app/apps/agent_app/runtime_request_builder.py
api/core/rbac/entities.py
api/core/tools/__base/tool_runtime.py
api/core/workflow/node_runtime.py
api/core/workflow/nodes/agent_v2/runtime_request_builder.py
api/extensions/ext_celery.py
api/extensions/ext_commands.py
api/models/__init__.py
api/services/account_service.py
api/services/agent_tool_inner_service.py
api/services/enterprise/rbac_service.py
api/services/entities/agent_tool_inner.py
)
for path in "${production_touchpoints[@]}"; do
add_production_target "$path"
done
printf '%s\0' "${production_touchpoints[@]}" > "$target_dir/glue-files"
if ((${#production_targets[@]} == 0)); then
echo "Dify KnowledgeFS production target set is empty" >&2
exit 1
fi
printf '%s\0' "${production_targets[@]}" > "$target_dir/production-files"
test_targets=()
add_test_target() {
local path="$1"
if [[ ! -f "$path" ]]; then
echo "required Dify KnowledgeFS unit test is missing: $path" >&2
exit 1
fi
test_targets+=("$path")
}
while IFS= read -r -d '' path; do
if [[ "$path" == *knowledge_fs* ]]; then
add_test_target "$path"
fi
done < <(find api/tests/unit_tests -type f -name '*.py' -print0)
test_touchpoints=(
api/tests/unit_tests/controllers/console/datasets/test_datasets.py
api/tests/unit_tests/controllers/console/test_wraps.py
api/tests/unit_tests/controllers/console/workspace/test_rbac.py
api/tests/unit_tests/core/agent/test_base_agent_runner.py
api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py
api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py
api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_runtime.py
api/tests/unit_tests/core/workflow/test_node_runtime.py
api/tests/unit_tests/services/enterprise/test_rbac_service.py
api/tests/unit_tests/services/test_account_service.py
api/tests/unit_tests/services/test_agent_tool_inner_service.py
)
for path in "${test_touchpoints[@]}"; do
add_test_target "$path"
done
required_test_scopes=(
/commands/
/configs/
/controllers/
/core/agent/
/core/app/apps/agent_app/
/core/tools/builtin_tool/providers/knowledge_fs/
/core/workflow/
/dev/
/extensions/
/migrations/
/models/
/repositories/
/services/
/tasks/
)
for required_scope in "${required_test_scopes[@]}"; do
scope_found=false
for path in "${test_targets[@]}"; do
if [[ "$path" == *"$required_scope"* ]]; then
scope_found=true
break
fi
done
if [[ "$scope_found" != true ]]; then
echo "required Dify KnowledgeFS test scope is empty: $required_scope" >&2
exit 1
fi
done
if ((${#test_targets[@]} == 0)); then
echo "Dify KnowledgeFS unit test target set is empty" >&2
exit 1
fi
printf '%s\0' "${test_targets[@]}" > "$target_dir/unit-test-files"
- name: Lint Dify KnowledgeFS integration
working-directory: .
run: |
set -euo pipefail
targets=()
while IFS= read -r -d '' path; do
targets+=("$path")
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/production-files"
if ((${#targets[@]} == 0)); then
echo "Dify KnowledgeFS production target manifest is empty" >&2
exit 1
fi
uv run --project api --dev ruff format --check "${targets[@]}"
uv run --project api --dev ruff check "${targets[@]}"
- name: Type-check Dify KnowledgeFS integration
working-directory: .
run: |
set -euo pipefail
targets=()
while IFS= read -r -d '' path; do
targets+=("$path")
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/production-files"
if ((${#targets[@]} == 0)); then
echo "Dify KnowledgeFS production target manifest is empty" >&2
exit 1
fi
PYREFLY_OUTPUT_FORMAT=github ./dev/pyrefly-check-local "${targets[@]}"
mypy_targets=()
for path in "${targets[@]}"; do
if [[ "$path" != api/migrations/* ]]; then
mypy_targets+=("${path#api/}")
fi
done
if ((${#mypy_targets[@]} == 0)); then
echo "Dify KnowledgeFS Mypy target set is empty" >&2
exit 1
fi
uv run --directory api --dev mypy \
--explicit-package-bases \
--exclude-gitignore \
--exclude '(^|/)conftest\.py$' \
--exclude 'tests/' \
--exclude 'migrations/' \
--check-untyped-defs \
--disable-error-code=import-untyped \
"${mypy_targets[@]}"
- name: Test Dify KnowledgeFS unit surface
working-directory: .
env:
COVERAGE_FILE: ${{ runner.temp }}/dify-knowledge-fs.coverage
run: |
set -euo pipefail
targets=()
while IFS= read -r -d '' path; do
targets+=("$path")
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/unit-test-files"
if ((${#targets[@]} == 0)); then
echo "Dify KnowledgeFS unit test target manifest is empty" >&2
exit 1
fi
uv run --project api --dev coverage run --branch --source=api -m pytest "${targets[@]}" --no-cov -q
- name: Enforce Dify KnowledgeFS focused coverage
working-directory: .
env:
COVERAGE_FILE: ${{ runner.temp }}/dify-knowledge-fs.coverage
KNOWLEDGE_FS_COVERAGE_BASE: >-
${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before || '' }}
run: |
set -euo pipefail
report="${RUNNER_TEMP:?}/dify-knowledge-fs-coverage.json"
coverage_base="$KNOWLEDGE_FS_COVERAGE_BASE"
if [[ "$GITHUB_EVENT_NAME" == 'push' && "$GITHUB_REF" == 'refs/heads/deploy/konwledge' ]]; then
latest_merge="$(git rev-list --first-parent --merges --max-count=1 "${coverage_base}..HEAD")"
if [[ -n "$latest_merge" ]]; then
# Mainline syncs are already gated on main; cover deployment-branch
# changes made after the newest sync instead of the merged diff.
coverage_base="$latest_merge"
fi
fi
uv run --project api --dev coverage json --show-contexts -o "$report"
uv run --project api --dev python api/dev/check_knowledge_fs_coverage.py \
--coverage-json "$report" \
--glue-manifest "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/glue-files" \
--base "$coverage_base" \
--minimum 90 \
--glue-minimum 90
- name: Verify Dify KnowledgeFS contract
working-directory: .
run: uv run --project api python api/dev/generate_knowledge_fs_contract.py --check
- name: Verify Dify Agent dependency lock
working-directory: .
run: uv lock --project dify-agent --check
- name: Install Dify Agent gate dependencies
working-directory: .
run: uv sync --project dify-agent --locked --dev
- name: Lint Dify Agent KnowledgeFS integration
working-directory: ./dify-agent
run: |
uv run --project . --dev ruff format --check \
src/dify_agent/layers/dify_core_tools/client.py \
tests/local/dify_agent/layers/dify_core_tools/test_client.py
uv run --project . --dev ruff check \
src/dify_agent/layers/dify_core_tools/client.py \
tests/local/dify_agent/layers/dify_core_tools/test_client.py
- name: Type-check Dify Agent KnowledgeFS integration
working-directory: ./dify-agent
run: >-
uv run --project . --dev pyrefly check --summary=none
src/dify_agent/layers/dify_core_tools/client.py
tests/local/dify_agent/layers/dify_core_tools/test_client.py
- name: Test Dify Agent KnowledgeFS integration
working-directory: ./dify-agent
run: >-
uv run --project . --dev python -m pytest
tests/local/dify_agent/layers/dify_core_tools/test_client.py
-q
skip:
name: Skip KnowledgeFS quality and contract gates
needs: check-changes
if: needs.check-changes.outputs.knowledge-fs != 'true' && github.event_name != 'workflow_dispatch'
runs-on: depot-ubuntu-24.04-4
steps:
- name: Report skipped KnowledgeFS checks
run: echo "No KnowledgeFS-related changes detected; skipping KnowledgeFS checks."
final:
name: KnowledgeFS CI
if: ${{ always() }}
needs:
- check-changes
- build
- quality
- skip
runs-on: depot-ubuntu-24.04-4
steps:
- name: Finalize KnowledgeFS CI status
env:
EVENT_NAME: ${{ github.event_name }}
BUILD_RESULT: ${{ needs.build.result }}
KNOWLEDGE_FS_CHANGED: ${{ needs.check-changes.outputs.knowledge-fs }}
QUALITY_RESULT: ${{ needs.quality.result }}
SKIP_RESULT: ${{ needs.skip.result }}
run: |
if [[ "$EVENT_NAME" == 'workflow_dispatch' || "$KNOWLEDGE_FS_CHANGED" == 'true' ]]; then
if [[ "$BUILD_RESULT" == 'success' && "$QUALITY_RESULT" == 'success' ]]; then
echo "KnowledgeFS build and checks ran successfully."
exit 0
fi
echo "KnowledgeFS build or checks failed: build=$BUILD_RESULT quality=$QUALITY_RESULT" >&2
exit 1
fi
if [[ "$SKIP_RESULT" == 'success' ]]; then
echo "KnowledgeFS checks were skipped because no related files changed."
exit 0
fi
echo "KnowledgeFS change detection or skip reporting failed with result: $SKIP_RESULT" >&2
exit 1

View File

@@ -15,10 +15,8 @@ permissions:
statuses: write
concurrency:
# Pull request deduplication is owned by skip-duplicate-actions so pre_job can
# publish should_skip before required status finalizers run.
group: main-ci-${{ github.event.merge_group.head_sha || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'merge_group' }}
group: main-ci-${{ github.event.pull_request.number || github.event.merge_group.head_sha || github.run_id }}
cancel-in-progress: true
jobs:
pre_job:
@@ -37,6 +35,8 @@ jobs:
# Check which paths were changed to determine which tests to run
check-changes:
name: Check Changed Files
needs: pre_job
if: needs.pre_job.outputs.should_skip != 'true'
runs-on: depot-ubuntu-24.04
outputs:
api-changed: ${{ steps.changes.outputs.api }}
@@ -46,12 +46,6 @@ jobs:
vdb-changed: ${{ steps.changes.outputs.vdb }}
migration-changed: ${{ steps.changes.outputs.migration }}
sandbox-runtime-changed: ${{ steps.changes.outputs.sandbox-runtime }}
dify-agent-changed: ${{ steps.changes.outputs.dify-agent }}
python-style-changed: ${{ steps.changes.outputs.python-style }}
dify-agent-style-changed: ${{ steps.changes.outputs.dify-agent-style }}
web-style-changed: ${{ steps.changes.outputs.web-style }}
ts-common-style-changed: ${{ steps.changes.outputs.ts-common-style }}
superlinter-changed: ${{ steps.changes.outputs.superlinter }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
@@ -85,6 +79,7 @@ jobs:
- 'pnpm-workspace.yaml'
- 'lint.config.ts'
- '.npmrc'
- '.nvmrc'
- '.github/workflows/cli-tests.yml'
- '.github/actions/setup-web/**'
web:
@@ -93,21 +88,23 @@ jobs:
- 'package.json'
- 'pnpm-lock.yaml'
- 'pnpm-workspace.yaml'
- '.nvmrc'
- '.github/workflows/main-ci.yml'
- '.github/workflows/web-tests.yml'
- '.github/actions/setup-web/**'
e2e:
- 'api/**'
- 'api/pyproject.toml'
- 'api/uv.lock'
- 'e2e/**'
- 'web/**'
- 'packages/**'
- 'package.json'
- 'pnpm-lock.yaml'
- 'pnpm-workspace.yaml'
- '.nvmrc'
- 'docker/docker-compose.middleware.yaml'
- 'docker/envs/middleware.env.example'
- 'docker/ssrf_proxy/**'
- 'docker/volumes/sandbox/conf/**'
- '.github/workflows/web-e2e.yml'
- '.github/workflows/main-ci.yml'
- '.github/actions/setup-web/**'
@@ -139,9 +136,6 @@ jobs:
sandbox-runtime:
- 'dify-agent-runtime/**'
- '.github/workflows/sandbox-runtime-tests.yml'
dify-agent:
- 'dify-agent/**'
- '.github/workflows/dify-agent-tests.yml'
migration:
- 'api/migrations/**'
- 'api/.env.example'
@@ -153,58 +147,6 @@ jobs:
- 'docker/generate_docker_compose'
- 'docker/ssrf_proxy/**'
- 'docker/volumes/sandbox/conf/**'
python-style:
- 'api/**'
- 'scripts/ast_grep_guard.py'
- 'scripts/check_no_new_getattr.py'
- 'scripts/check_no_new_controller_sqlalchemy.py'
- 'scripts/lint_controller_sqlalchemy.py'
- 'scripts/ast_grep_rules/no_new_getattr.yml'
- 'scripts/ast_grep_rules/no_new_controller_sqlalchemy.yml'
- '.github/workflows/style.yml'
- '.github/workflows/main-ci.yml'
dify-agent-style:
- 'dify-agent/**'
- '.github/workflows/style.yml'
- '.github/workflows/main-ci.yml'
web-style:
- 'web/**'
- 'e2e/**'
- 'sdks/nodejs-client/**'
- 'packages/**'
- 'package.json'
- 'pnpm-lock.yaml'
- 'pnpm-workspace.yaml'
- 'knip.config.ts'
- 'scripts/check-web-production-unused-after-knip-fix.mjs'
- '.github/workflows/style.yml'
- '.github/actions/setup-web/**'
ts-common-style:
- 'web/**'
- 'cli/**'
- 'e2e/**'
- 'sdks/nodejs-client/**'
- 'packages/**'
- 'package.json'
- 'pnpm-lock.yaml'
- 'pnpm-workspace.yaml'
- 'vite.config.ts'
- 'lint.config.ts'
- 'eslint.config.mjs'
- 'knip.config.ts'
- 'scripts/check-web-production-unused-after-knip-fix.mjs'
- 'oxlint-suppressions.json'
- 'eslint-suppressions.json'
- '.vscode/**'
- '.github/**'
superlinter:
- '**.sh'
- '**.yaml'
- '**.yml'
- '**Dockerfile'
- 'dev/**'
- '.editorconfig'
- '.vite-hooks/**'
# Run tests in parallel while always emitting stable required checks.
api-tests-run:
@@ -451,20 +393,11 @@ jobs:
style-check:
name: Style Check
if: ${{ always() }}
needs:
- pre_job
- check-changes
needs: pre_job
if: needs.pre_job.outputs.should_skip != 'true'
uses: ./.github/workflows/style.yml
with:
base-rev: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
classification-succeeded: ${{ needs.check-changes.result == 'success' }}
run-python-style: ${{ needs.check-changes.outputs.python-style-changed == 'true' }}
run-dify-agent-style: ${{ needs.check-changes.outputs.dify-agent-style-changed == 'true' }}
run-web-style: ${{ needs.check-changes.outputs.web-style-changed == 'true' }}
run-ts-common-style: ${{ needs.check-changes.outputs.ts-common-style-changed == 'true' }}
run-superlinter: ${{ needs.check-changes.outputs.superlinter-changed == 'true' }}
skip-checks: ${{ needs.pre_job.outputs.should_skip == 'true' }}
vdb-tests-run:
name: Run VDB Tests
@@ -643,63 +576,3 @@ jobs:
echo "Sandbox runtime tests were not required, but the skip job finished with result: $SKIP_RESULT" >&2
exit 1
dify-agent-tests-run:
name: Run Dify Agent Tests
needs:
- pre_job
- check-changes
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.dify-agent-changed == 'true'
uses: ./.github/workflows/dify-agent-tests.yml
secrets: inherit
dify-agent-tests-skip:
name: Skip Dify Agent Tests
needs:
- pre_job
- check-changes
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.dify-agent-changed != 'true'
runs-on: depot-ubuntu-24.04
steps:
- name: Report skipped dify agent tests
run: echo "No dify-agent-related changes detected; skipping dify agent tests."
dify-agent-tests:
name: Dify Agent Tests
if: ${{ always() }}
needs:
- pre_job
- check-changes
- dify-agent-tests-run
- dify-agent-tests-skip
runs-on: depot-ubuntu-24.04
steps:
- name: Finalize Dify Agent Tests status
env:
SHOULD_SKIP_WORKFLOW: ${{ needs.pre_job.outputs.should_skip }}
TESTS_CHANGED: ${{ needs.check-changes.outputs.dify-agent-changed }}
RUN_RESULT: ${{ needs.dify-agent-tests-run.result }}
SKIP_RESULT: ${{ needs.dify-agent-tests-skip.result }}
run: |
if [[ "$SHOULD_SKIP_WORKFLOW" == 'true' ]]; then
echo "Dify agent tests were skipped because this workflow run duplicated a successful or newer run."
exit 0
fi
if [[ "$TESTS_CHANGED" == 'true' ]]; then
if [[ "$RUN_RESULT" == 'success' ]]; then
echo "Dify agent tests ran successfully."
exit 0
fi
echo "Dify agent tests were required but finished with result: $RUN_RESULT" >&2
exit 1
fi
if [[ "$SKIP_RESULT" == 'success' ]]; then
echo "Dify agent tests were skipped because no dify-agent-related files changed."
exit 0
fi
echo "Dify agent tests were not required, but the skip job finished with result: $SKIP_RESULT" >&2
exit 1

View File

@@ -39,13 +39,12 @@ jobs:
- 'e2e/tsx-register.js'
- 'package.json'
- 'pnpm-lock.yaml'
- '.nvmrc'
- '.github/workflows/post-merge.yml'
- '.github/workflows/web-e2e.yml'
- '.github/actions/setup-web/**'
- 'docker/docker-compose.middleware.yaml'
- 'docker/envs/middleware.env.example'
- 'docker/ssrf_proxy/**'
- 'docker/volumes/sandbox/conf/**'
- 'dify-agent/**'
- 'dify-agent-runtime/**'
- 'api/pyproject.toml'
@@ -55,30 +54,12 @@ jobs:
- 'api/core/app/apps/agent_app/**'
- 'api/core/workflow/nodes/agent_v2/**'
- 'api/controllers/console/agent/**'
- 'api/controllers/console/app/agent*.py'
- 'api/controllers/console/app/audio.py'
- 'api/controllers/common/agent_access.py'
- 'api/controllers/inner_api/agent/**'
- 'api/controllers/inner_api/wraps.py'
- 'api/controllers/service_api/app/completion.py'
- 'api/controllers/service_api/wraps.py'
- 'api/controllers/web/completion.py'
- 'api/controllers/web/wraps.py'
- 'api/services/agent_llm_inner_service.py'
- 'api/services/agent_tool_inner_service.py'
- 'api/services/audio_service.py'
- 'api/services/entities/agent_llm_inner.py'
- 'api/services/entities/agent_tool_inner.py'
- 'api/services/errors/agent_tool_inner.py'
- 'api/services/agent/**'
- 'api/core/plugin/**'
- 'api/services/plugin/**'
- 'api/core/tools/**'
- 'api/services/tools/**'
- 'packages/contracts/package.json'
- 'packages/contracts/console.ts'
- 'packages/contracts/generated/api/console/router.gen.ts'
- 'packages/contracts/generated/api/web/**'
- 'packages/contracts/generated/api/console/agent/**'
- 'packages/contracts/generated/api/console/apps/**'
- 'packages/contracts/generated/api/console/datasets/**'
@@ -86,21 +67,20 @@ jobs:
- 'packages/contracts/generated/api/console/workspaces/**'
- 'packages/contracts/generated/api/service/**'
- 'web/features/agent-v2/**'
- 'web/app/\(commonLayout\)/agents/**'
- 'web/app/\(commonLayout\)/@detailSidebar/agents/**'
- 'web/app/\(shareLayout\)/chat/**'
- 'web/app/\(shareLayout\)/components/**'
- 'web/app/components/base/chat/chat-with-history/**'
- 'web/app/(commonLayout)/agents/**'
- 'web/app/(commonLayout)/@detailSidebar/agents/**'
- 'web/app/(commonLayout)/roster/**'
- 'web/app/components/base/chat/chat/**'
- 'web/app/components/base/voice-input/**'
- 'web/app/components/workflow/nodes/agent-v2/**'
- 'web/i18n/en-US/agent-v-2.json'
- 'web/i18n/en-US/common.json'
- 'web/service/base.ts'
- 'web/service/console/**'
- 'web/service/client.ts'
- 'web/service/console-link.ts'
- 'web/service/console-openapi-url.ts'
- 'web/service/console-router-loader.ts'
- 'web/service/share.ts'
- 'web/service/use-share.ts'
- 'web/service/webapp-address.ts'
- 'web/package.json'
- 'pnpm-workspace.yaml'

View File

@@ -22,7 +22,7 @@ jobs:
fetch-depth: 0
- name: Setup Python & UV
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true

View File

@@ -24,7 +24,7 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Setup Python & UV
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true

View File

@@ -22,7 +22,7 @@ jobs:
fetch-depth: 0
- name: Setup Python & UV
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true

View File

@@ -6,35 +6,6 @@ on:
base-rev:
required: true
type: string
classification-succeeded:
description: Whether the caller successfully classified the changed files.
required: true
type: boolean
run-python-style:
description: Run Python style checks for the changed files.
required: true
type: boolean
run-dify-agent-style:
description: Run Dify Agent style checks for the changed files.
required: true
type: boolean
run-web-style:
description: Run Web style checks for the changed files.
required: true
type: boolean
run-ts-common-style:
description: Run shared TypeScript style checks for the changed files.
required: true
type: boolean
run-superlinter:
description: Run SuperLinter for the changed files.
required: true
type: boolean
skip-checks:
description: Create the required check runs without repeating previously successful work.
required: false
type: boolean
default: false
permissions:
checks: write
@@ -44,72 +15,70 @@ permissions:
jobs:
python-style:
name: Python Style
if: ${{ !inputs.skip-checks && (!inputs.classification-succeeded || inputs.run-python-style || inputs.run-dify-agent-style) }}
runs-on: depot-ubuntu-24.04
steps:
- name: Verify changed-file classification
if: ${{ !inputs.classification-succeeded }}
run: |
echo "Changed-file classification failed; refusing to skip Python style checks." >&2
exit 1
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
fetch-depth: 0
- name: Check changed files
id: changed-files
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
with:
files: |
api/**
scripts/ast_grep_guard.py
scripts/check_no_new_getattr.py
scripts/check_no_new_controller_sqlalchemy.py
scripts/lint_controller_sqlalchemy.py
scripts/ast_grep_rules/no_new_getattr.yml
scripts/ast_grep_rules/no_new_controller_sqlalchemy.yml
.github/workflows/style.yml
.github/workflows/main-ci.yml
- name: Setup UV and Python
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
if: steps.changed-files.outputs.any_changed == 'true'
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: false
python-version: '3.12'
cache-dependency-glob: |
api/uv.lock
dify-agent/uv.lock
cache-dependency-glob: api/uv.lock
- name: Install dependencies
if: inputs.run-python-style
if: steps.changed-files.outputs.any_changed == 'true'
run: uv sync --project api --dev
- name: Run Import Linter
if: inputs.run-python-style
if: steps.changed-files.outputs.any_changed == 'true'
run: uv run --directory api --dev lint-imports
- name: Run Response Contract Linter
if: inputs.run-python-style
if: steps.changed-files.outputs.any_changed == 'true'
run: uv run --project api --dev python api/dev/lint_response_contracts.py --fail-on-mismatch
- name: Run No New Getattr Guard
if: inputs.run-python-style
if: steps.changed-files.outputs.any_changed == 'true'
run: uv run --project api python scripts/check_no_new_getattr.py --base-rev "${{ inputs.base-rev }}"
- name: Run No New Controller SQLAlchemy Guard
if: inputs.run-python-style
if: steps.changed-files.outputs.any_changed == 'true'
run: uv run --project api python scripts/check_no_new_controller_sqlalchemy.py --base-rev "${{ inputs.base-rev }}"
- name: Run Type Checks
if: inputs.run-python-style
if: steps.changed-files.outputs.any_changed == 'true'
env:
PYREFLY_OUTPUT_FORMAT: github
run: make type-check-core
- name: Run Dify Agent Ruff
if: inputs.run-dify-agent-style
run: make -C dify-agent check
- name: Run Dify Agent Type Checks
if: inputs.run-dify-agent-style
run: make -C dify-agent typecheck
- name: Dotenv check
if: inputs.run-python-style
if: steps.changed-files.outputs.any_changed == 'true'
run: uv run --project api dotenv-linter ./api/.env.example ./web/.env.example
web-style:
name: Web Style
if: ${{ !inputs.skip-checks && (!inputs.classification-succeeded || inputs.run-web-style) }}
runs-on: depot-ubuntu-24.04
defaults:
run:
@@ -119,93 +88,131 @@ jobs:
pull-requests: read
steps:
- name: Verify changed-file classification
if: ${{ !inputs.classification-succeeded }}
working-directory: .
run: |
echo "Changed-file classification failed; refusing to skip Web style checks." >&2
exit 1
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Check changed files
id: changed-files
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
with:
files: |
web/**
e2e/**
sdks/nodejs-client/**
packages/**
package.json
pnpm-lock.yaml
pnpm-workspace.yaml
knip.config.ts
scripts/check-web-production-unused-after-knip-fix.mjs
.nvmrc
.github/workflows/style.yml
.github/actions/setup-web/**
- name: Setup web environment
if: steps.changed-files.outputs.any_changed == 'true'
uses: ./.github/actions/setup-web
- name: Web tsslint
if: steps.changed-files.outputs.any_changed == 'true'
env:
NODE_OPTIONS: --max-old-space-size=4096
run: vp run lint:tss
- name: Web dead code check
if: steps.changed-files.outputs.any_changed == 'true'
working-directory: .
run: vp run knip
- name: Web dead code check production
if: steps.changed-files.outputs.any_changed == 'true'
working-directory: .
run: vp run knip:production
- name: Web production unused declarations check
if: steps.changed-files.outputs.any_changed == 'true'
working-directory: .
run: vp run knip:production-unused-check
ts-common-style:
name: TS Common
if: ${{ !inputs.skip-checks && (!inputs.classification-succeeded || inputs.run-ts-common-style) }}
runs-on: depot-ubuntu-24.04-4
permissions:
checks: write
pull-requests: read
steps:
- name: Verify changed-file classification
if: ${{ !inputs.classification-succeeded }}
run: |
echo "Changed-file classification failed; refusing to skip TypeScript style checks." >&2
exit 1
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Check changed files
id: changed-files
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
with:
files: |
web/**
cli/**
e2e/**
sdks/nodejs-client/**
packages/**
package.json
pnpm-lock.yaml
pnpm-workspace.yaml
.nvmrc
vite.config.ts
lint.config.ts
eslint.config.mjs
oxlint-suppressions.json
eslint-suppressions.json
.vscode/**
.github/**
- name: Setup web environment
if: steps.changed-files.outputs.any_changed == 'true'
uses: ./.github/actions/setup-web
- name: Static check
run: vp run -w check
if: steps.changed-files.outputs.any_changed == 'true'
run: pnpm -w check
superlinter:
name: SuperLinter
if: ${{ !inputs.skip-checks && (!inputs.classification-succeeded || inputs.run-superlinter) }}
runs-on: depot-ubuntu-24.04
steps:
- name: Verify changed-file classification
if: ${{ !inputs.classification-succeeded }}
run: |
echo "Changed-file classification failed; refusing to skip SuperLinter." >&2
exit 1
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Check changed files
id: changed-files
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
with:
files: |
**.sh
**.yaml
**.yml
**Dockerfile
dev/**
.editorconfig
- name: Super-linter
uses: super-linter/super-linter/slim@4ce20838b8ab83717e78138c5b3a1407148e0918 # v8.7.0
if: steps.changed-files.outputs.any_changed == 'true'
env:
BASH_SEVERITY: warning
DEFAULT_BRANCH: origin/main
EDITORCONFIG_FILE_NAME: editorconfig-checker.json
ENABLE_GITHUB_PULL_REQUEST_SUMMARY_COMMENT: false
FILTER_REGEX_INCLUDE: pnpm-lock.yaml
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
IGNORE_GENERATED_FILES: true
IGNORE_GITIGNORED_FILES: true
VALIDATE_ALL_CODEBASE: false
VALIDATE_BASH: true
VALIDATE_BASH_EXEC: true
# FIXME: temporarily disabled until api-docker.yaml's run script is fixed for shellcheck

View File

@@ -31,7 +31,7 @@ jobs:
- name: Use Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version-file: package.json
node-version: 22
cache: ''
cache-dependency-path: 'pnpm-lock.yaml'

View File

@@ -8,19 +8,15 @@ on:
workflow_dispatch:
inputs:
files:
description: 'Specific files to translate (space-separated, e.g., "app common").
Required for full mode; leave empty in incremental mode to use en-US
files changed since HEAD~1.'
description: 'Specific files to translate (space-separated, e.g., "app common"). Required for full mode; leave empty in incremental mode to use en-US files changed since HEAD~1.'
required: false
type: string
languages:
description: 'Specific languages to translate (space-separated, e.g., "zh-Hans
ja-JP"). Leave empty for all supported target languages except en-US.'
description: 'Specific languages to translate (space-separated, e.g., "zh-Hans ja-JP"). Leave empty for all supported target languages except en-US.'
required: false
type: string
mode:
description: 'Sync mode: incremental (compare with previous en-US revision) or
full (sync all keys in scope)'
description: 'Sync mode: incremental (compare with previous en-US revision) or full (sync all keys in scope)'
required: false
default: incremental
type: choice
@@ -162,7 +158,7 @@ jobs:
- name: Run Claude Code for Translation Sync
if: steps.context.outputs.CHANGED_FILES != ''
uses: anthropics/claude-code-action@d75b94d5ad426cb8546e6628b6f5f19b84e5cce1 # v1.0.216
uses: anthropics/claude-code-action@c038e4dcdedfbbca18dfb17df35a17e40ded4ddc # v1.0.186
with:
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
github_token: ${{ secrets.GITHUB_TOKEN }}

View File

@@ -29,14 +29,14 @@ jobs:
persist-credentials: false
- name: Free Disk Space
uses: endersonmenezes/free-disk-space@2a22f8c59cae9cddae7194b33e6b92d023b2b4b8 # v4.0.0
uses: endersonmenezes/free-disk-space@7901478139cff6e9d44df5972fd8ab8fcade4db1 # v3.2.2
with:
remove_dotnet: true
remove_haskell: true
remove_tool_cache: true
- name: Setup UV and Python
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true
python-version: ${{ matrix.python-version }}

View File

@@ -26,14 +26,14 @@ jobs:
persist-credentials: false
- name: Free Disk Space
uses: endersonmenezes/free-disk-space@2a22f8c59cae9cddae7194b33e6b92d023b2b4b8 # v4.0.0
uses: endersonmenezes/free-disk-space@7901478139cff6e9d44df5972fd8ab8fcade4db1 # v3.2.2
with:
remove_dotnet: true
remove_haskell: true
remove_tool_cache: true
- name: Setup UV and Python
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true
python-version: ${{ matrix.python-version }}

View File

@@ -12,196 +12,13 @@ permissions:
contents: read
jobs:
core-build:
name: Prepare Core E2E Web Build
if: ${{ !inputs.run-external-runtime }}
runs-on: depot-ubuntu-24.04-4
timeout-minutes: 30
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Setup web dependencies
uses: ./.github/actions/setup-web
- name: Run E2E support unit tests
working-directory: ./e2e
run: vp run test:unit
- name: Build production Web app
working-directory: ./e2e
env:
E2E_FORCE_WEB_BUILD: '1'
run: vp run e2e:web:build
- name: Package Web build
run: >-
tar -cf e2e-web-build.tar -C web
.next/BUILD_ID
.next/e2e-web-build.sha256
.next/standalone
.next/static
- name: Upload Web build
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: core-e2e-web-build
path: e2e-web-build.tar
if-no-files-found: error
overwrite: true
retention-days: 7
chromium-full:
name: Chromium Full (${{ matrix.shard }}/3)
if: ${{ !inputs.run-external-runtime }}
needs: core-build
runs-on: ubuntu-24.04
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
shard: [1, 2, 3]
env:
E2E_ADMIN_EMAIL: e2e-admin@example.com
E2E_ADMIN_NAME: E2E Admin
E2E_ADMIN_PASSWORD: E2eAdmin12345
E2E_INIT_PASSWORD: E2eInit12345
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Setup web dependencies
uses: ./.github/actions/setup-web
- name: Setup UV and Python
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
with:
enable-cache: true
python-version: '3.12'
cache-dependency-glob: api/uv.lock
- name: Install API dependencies
run: uv sync --project api --dev
- name: Download Web build
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: core-e2e-web-build
- name: Extract Web build
run: tar -xf e2e-web-build.tar -C web
- name: Install Chromium
timeout-minutes: 15
working-directory: ./e2e
run: vp run e2e:install:ci:chromium
- name: Run Chromium full shard
working-directory: ./e2e
run: vp run e2e:full -- --shard ${{ matrix.shard }}/3
- name: Upload Cucumber report
if: ${{ !cancelled() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: cucumber-report-chromium-${{ matrix.shard }}
path: e2e/cucumber-report
if-no-files-found: ignore
overwrite: true
retention-days: 7
- name: Upload E2E logs
if: ${{ !cancelled() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-logs-chromium-${{ matrix.shard }}
path: e2e/.logs/*.log
if-no-files-found: ignore
include-hidden-files: true
overwrite: true
retention-days: 7
webkit-smoke:
name: WebKit Browser Smoke
if: ${{ !inputs.run-external-runtime }}
needs: core-build
runs-on: ubuntu-24.04
timeout-minutes: 30
env:
E2E_ADMIN_EMAIL: e2e-admin@example.com
E2E_ADMIN_NAME: E2E Admin
E2E_ADMIN_PASSWORD: E2eAdmin12345
E2E_BROWSER: webkit
E2E_INIT_PASSWORD: E2eInit12345
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Setup web dependencies
uses: ./.github/actions/setup-web
- name: Setup UV and Python
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
with:
enable-cache: true
python-version: '3.12'
cache-dependency-glob: api/uv.lock
- name: Install API dependencies
run: uv sync --project api --dev
- name: Download Web build
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: core-e2e-web-build
- name: Extract Web build
run: tar -xf e2e-web-build.tar -C web
- name: Install WebKit
timeout-minutes: 15
working-directory: ./e2e
run: vp run e2e:install:ci:webkit
- name: Run WebKit browser smoke
working-directory: ./e2e
run: vp run e2e:full -- --tags '@browser-smoke'
- name: Upload Cucumber report
if: ${{ !cancelled() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: cucumber-report-webkit
path: e2e/cucumber-report
if-no-files-found: ignore
overwrite: true
retention-days: 7
- name: Upload E2E logs
if: ${{ !cancelled() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-logs-webkit
path: e2e/.logs/*.log
if-no-files-found: ignore
include-hidden-files: true
overwrite: true
retention-days: 7
external-runtime:
test:
name: Web Full-Stack E2E
if: ${{ inputs.run-external-runtime }}
runs-on: ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
timeout-minutes: 120
defaults:
run:
shell: bash
steps:
- name: Checkout code
@@ -209,11 +26,16 @@ jobs:
with:
persist-credentials: false
- name: Switch Ubuntu mirror
run: |
sudo sed -i 's|http://us-east-1.ec2.archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list.d/ubuntu.sources
sudo apt-get update
- name: Setup web dependencies
uses: ./.github/actions/setup-web
- name: Setup UV and Python
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true
python-version: '3.12'
@@ -224,12 +46,82 @@ jobs:
- name: Install API dependencies
run: uv sync --project api --dev
- name: Install Chromium
- name: Run E2E support unit tests
if: ${{ !inputs.run-external-runtime }}
working-directory: ./e2e
run: vp run test:unit
- name: Install Playwright browsers for core E2E
if: ${{ !inputs.run-external-runtime }}
timeout-minutes: 15
working-directory: ./e2e
run: vp run e2e:install:ci
- name: Install Chromium for external runtime E2E
if: ${{ inputs.run-external-runtime }}
timeout-minutes: 15
working-directory: ./e2e
run: vp run e2e:install:ci:chromium
- name: Run isolated source-api and built-web Cucumber E2E tests
if: ${{ !inputs.run-external-runtime }}
working-directory: ./e2e
env:
E2E_ADMIN_EMAIL: e2e-admin@example.com
E2E_ADMIN_NAME: E2E Admin
E2E_ADMIN_PASSWORD: E2eAdmin12345
E2E_FORCE_WEB_BUILD: '1'
E2E_INIT_PASSWORD: E2eInit12345
run: vp run e2e:full
- name: Preserve Chromium E2E report and logs
if: ${{ !cancelled() && !inputs.run-external-runtime }}
run: |
if [[ -d e2e/cucumber-report ]]; then
mv e2e/cucumber-report e2e/cucumber-report-non-external
fi
if [[ -d e2e/.logs ]]; then
mv e2e/.logs e2e/.logs-non-external
fi
- name: Run WebKit keyboard and browser smoke tests
if: ${{ !inputs.run-external-runtime }}
working-directory: ./e2e
env:
E2E_ADMIN_EMAIL: e2e-admin@example.com
E2E_ADMIN_NAME: E2E Admin
E2E_ADMIN_PASSWORD: E2eAdmin12345
E2E_BROWSER: webkit
E2E_INIT_PASSWORD: E2eInit12345
run: |
teardown_webkit_smoke() {
local run_status=$?
trap - EXIT
if ! vp run e2e:middleware:down; then
echo "::error title=E2E teardown failed::WebKit smoke middleware did not shut down cleanly."
if [[ "$run_status" -eq 0 ]]; then
run_status=1
fi
fi
exit "$run_status"
}
trap teardown_webkit_smoke EXIT
vp run e2e:middleware:up
vp run e2e -- --tags '@browser-smoke'
- name: Preserve WebKit E2E report and logs
if: ${{ !cancelled() && !inputs.run-external-runtime }}
run: |
if [[ -d e2e/cucumber-report ]]; then
mv e2e/cucumber-report e2e/cucumber-report-webkit
fi
if [[ -d e2e/.logs ]]; then
mv e2e/.logs e2e/.logs-webkit
fi
- name: Run prepared and external runtime E2E tests
if: ${{ inputs.run-external-runtime }}
working-directory: ./e2e
env:
E2E_ADMIN_EMAIL: e2e-admin@example.com
@@ -262,8 +154,10 @@ jobs:
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: cucumber-report
path: e2e/cucumber-report
overwrite: true
path: |
e2e/cucumber-report
e2e/cucumber-report-non-external
e2e/cucumber-report-webkit
retention-days: 7
- name: Upload E2E logs
@@ -271,17 +165,18 @@ jobs:
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-logs
path: e2e/.logs/*.log
path: |
e2e/.logs/*.log
e2e/.logs-non-external/*.log
e2e/.logs-webkit/*.log
include-hidden-files: true
overwrite: true
retention-days: 7
- name: Upload E2E seed report
if: ${{ !cancelled() }}
if: ${{ !cancelled() && inputs.run-external-runtime }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-seed-report
path: e2e/seed-report
if-no-files-found: ignore
overwrite: true
retention-days: 7

View File

@@ -33,7 +33,7 @@ jobs:
uses: ./.github/actions/setup-web
- name: Run tests
run: vp test run --project unit --reporter=blob --reporter=minimal --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --coverage
run: vp test run --reporter=blob --reporter=minimal --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --coverage
- name: Upload blob report
if: ${{ !cancelled() }}
@@ -44,41 +44,6 @@ jobs:
include-hidden-files: true
retention-days: 1
browser-test:
name: Web Browser Tests
runs-on: ubuntu-24.04
timeout-minutes: 20
defaults:
run:
shell: bash
working-directory: ./web
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Setup web environment
uses: ./.github/actions/setup-web
- name: Install Chromium for Browser Mode
timeout-minutes: 15
run: vp exec playwright install --with-deps --only-shell chromium
- name: Run browser tests
run: vp test run --project browser --silent=passed-only
- name: Upload browser failure artifacts
if: ${{ failure() && !cancelled() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: browser-test-failures
path: web/.vitest-browser/
if-no-files-found: ignore
include-hidden-files: true
retention-days: 7
merge-reports:
name: Merge Test Reports
if: ${{ !cancelled() }}
@@ -121,7 +86,7 @@ jobs:
dify-ui-test:
name: dify-ui Tests
runs-on: ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
timeout-minutes: 20
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
@@ -136,6 +101,11 @@ jobs:
with:
persist-credentials: false
- name: Switch Ubuntu mirror
run: |
sudo sed -i 's|http://us-east-1.ec2.archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list.d/ubuntu.sources
sudo apt-get update
- name: Setup web environment
uses: ./.github/actions/setup-web
@@ -146,16 +116,6 @@ jobs:
- name: Run dify-ui tests
run: vp test run --project unit --coverage --silent=passed-only
- name: Upload dify-ui test failure artifacts
if: ${{ failure() && !cancelled() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: dify-ui-test-failures
path: packages/dify-ui/.vitest-browser/
if-no-files-found: ignore
include-hidden-files: true
retention-days: 7
- name: Report coverage
if: ${{ env.CODECOV_TOKEN != '' }}
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
@@ -167,7 +127,7 @@ jobs:
dify-ui-storybook-test:
name: dify-ui Storybook Tests
runs-on: ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
timeout-minutes: 20
defaults:
run:
@@ -180,6 +140,11 @@ jobs:
with:
persist-credentials: false
- name: Switch Ubuntu mirror
run: |
sudo sed -i 's|http://us-east-1.ec2.archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list.d/ubuntu.sources
sudo apt-get update
- name: Setup web environment
uses: ./.github/actions/setup-web
@@ -189,13 +154,3 @@ jobs:
- name: Run dify-ui Storybook tests
run: vp run test:storybook
- name: Upload dify-ui Storybook test failure artifacts
if: ${{ failure() && !cancelled() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: dify-ui-storybook-test-failures
path: packages/dify-ui/.vitest-browser/
if-no-files-found: ignore
include-hidden-files: true
retention-days: 7

6
.gitignore vendored
View File

@@ -30,11 +30,6 @@ share/python-wheels/
*.egg
MANIFEST
# KnowledgeFS is an independently rooted TypeScript workspace. Its admin `lib`
# directory contains source files rather than Python build output.
!/knowledge-fs/apps/admin/lib/
!/knowledge-fs/apps/admin/lib/**
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
@@ -268,7 +263,6 @@ scripts/stress-test/reports/
.qoder/*
.context/
# Vitest local reports
web/.vitest-browser/
web/.vitest-reports/
# dify-agent-runtime

View File

@@ -1,30 +0,0 @@
# KnowledgeFS metadata catalog
## What changed
- Added a durable, per-knowledge-space metadata field catalog with `string`, `number`, and `time` field types.
- Added document-to-field bindings so field usage counts no longer require scanning every document.
- Added metadata field list, create, rename, and delete APIs in KnowledgeFS and the Dify console proxy.
- Made document metadata writes validate field definitions and update the document, bindings, and active retrieval asset in one database transaction.
- Made field rename and delete cascade to bound document values and active retrieval metadata.
- Updated the KnowledgeFS document UI to read and mutate the field catalog directly instead of deriving it from paginated document data.
- Included nested `userMetadata` in retrieval metadata filtering.
## Data migration
Migration `0040_knowledge_space_metadata` creates the catalog and binding tables and backfills consistently typed custom metadata already stored on logical documents. Reserved system keys and inconsistently typed values are left unbound rather than inventing an unsafe catalog type.
## Compatibility
- Existing document metadata remains readable.
- A custom field must exist in the space catalog before a new value can be assigned.
- Catalog mutations use row-version compare-and-swap semantics.
- No default value is written to every document when a field is created.
## Verification
- Database schema and migration suites.
- Metadata repository, handler, logical-document lifecycle, and retrieval candidate suites.
- Dify facade, product operation, and controller delegation suites.
- KnowledgeFS TypeScript type checking and Biome checks.
- Frontend metadata page tests, Vite+ checks, and TypeScript type checking.

View File

@@ -1,99 +0,0 @@
# KnowledgeFS unified error handling
## Goal
Replace the current mix of raw exceptions, generic strings, status-only proxy errors, and UI pass-through messages with one safe and actionable public error contract across KnowledgeFS, the Dify API BFF, background tasks, polling/SSE, and the New RAG UI.
The implementation must preserve internal diagnostic detail without exposing provider payloads, credentials, request headers, signed URLs, stack traces, or other unsafe values to product clients.
## Compatibility strategy
- Keep existing `code`/`error` HTTP fields and `errorCode`/`errorMessage` task fields during the transition.
- Add structured failure metadata rather than replacing existing fields in one breaking change.
- Persist stable error codes and safe fallback messages; localize at the Dify Web boundary.
- Derive retry policy and recommended actions from the central error catalog wherever possible, avoiding an initial database migration.
- Unknown exceptions always become a generic internal failure with a trace/support identifier; their original cause remains log-only.
## Iteration plan
### P0 — Inventory and contract design
- [x] Inventory existing public error schemas, task failure fields, retry behavior, BFF mappings, and UI consumers.
- [x] Define categories, retry policies, recommended actions, safe parameters, and transport naming.
- [x] Define backward-compatible HTTP and background-task failure schemas.
- [x] Document security rules for public versus internal diagnostic messages.
### P1 — KnowledgeFS error foundation
- [x] Add a central error catalog containing stable codes and public metadata.
- [x] Add a typed `KnowledgeFsError` and a boundary normalizer for typed domain errors, provider errors, aborts/timeouts, validation failures, and unknown exceptions.
- [x] Add helpers that serialize only allowlisted public metadata.
- [x] Retain existing validation/not-found classes as compatible typed specializations.
- [x] Add unit tests for catalog completeness, sanitization, retry policies, safe parameter filtering, and unknown errors.
### P2 — Document compilation and model preflight
- [x] Replace the generic “selected model could not be validated” collapse with stable actionable model errors.
- [x] Split model-not-found, credentials-invalid, validation-unavailable/timeout, capability mismatch, identity mismatch, and embedding-dimension failures.
- [x] Keep automatic retry behavior for transient failures and mark configuration failures as `after_configuration` or `never`.
- [x] Ensure persisted task messages are safe public fallbacks, not raw provider exceptions.
- [x] Add regression tests for each model/preflight outcome.
### P3 — Background tasks and streams
- [x] Add a structured `failure` object to document, source, and bulk task responses.
- [x] Derive `canRetry` from failure retry policy instead of terminal state alone.
- [x] Return the same failure object through list/polling and terminal SSE events.
- [x] Preserve legacy task fields for existing clients.
- [x] Add task mapping, retry-control, list response, and SSE parity tests.
### P4 — HTTP gateway
- [x] Expand the shared KnowledgeFS error response schema with safe message, retry policy, action, parameters, and trace ID.
- [x] Normalize typed errors in the global gateway handler.
- [x] Keep route-specific compatibility fields while removing raw exception exposure.
- [x] Add OpenAPI and handler tests for known, validation, not-found, timeout, and unknown errors.
### P5 — Dify API BFF
- [x] Parse and validate allowlisted KnowledgeFS public error payloads.
- [x] Preserve safe upstream code and metadata in `KnowledgeFSProductRequestRejectedError` instead of retaining only HTTP status.
- [x] Map the structured error into the Console API response while retaining existing generic fallbacks.
- [x] Keep malformed or unrecognized upstream payloads masked.
- [x] Add remote-client, service, controller, and DTO tests.
### P6 — Web product experience
- [x] Add a feature-owned error resolver using generated contract types.
- [x] Add localized messages for every public error code in all supported locales.
- [x] Render safe localized messages instead of raw backend strings.
- [x] Select Retry, Configure model, Configure parser, Re-upload, or Contact administrator actions from the failure contract.
- [x] Preserve accessible button names and existing task interaction behavior.
- [x] Add focused unit/component tests for permanent, configuration-dependent, transient, and unknown failures.
### P7 — Remaining major boundaries
- [x] Route upload, parser, storage, source, research, query, metadata, and authorization boundary failures through the common normalizer.
- [x] Reuse the existing safe source-operation allowlist through the common catalog.
- [x] Remove direct public use of provider/runtime error messages at migrated boundaries.
- [x] Add code-health coverage preventing new unregistered public task error codes and unsafe raw-message pass-through.
### P8 — Contracts, documentation, and verification
- [x] Regenerate KnowledgeFS OpenAPI/capability artifacts and Dify generated contracts.
- [x] Update the KnowledgeFS contract lock intentionally after reviewing subtree changes.
- [x] Document public error codes, retry semantics, and client behavior.
- [x] Run targeted KnowledgeFS, API, and Web tests plus type/lint/contract gates.
- [x] Review the final diff without staging or overwriting unrelated workspace changes.
## Acceptance criteria
- Every migrated public failure has a stable registered code, safe fallback message, category, retry policy, and recommended action.
- Model configuration failures no longer display the same generic English sentence.
- A permanent or configuration-dependent failure never presents a misleading immediate Retry action.
- Polling and SSE expose equivalent failure metadata.
- Dify API preserves valid safe KnowledgeFS error codes instead of reducing all 4xx responses to HTTP status alone.
- Web displays localized actionable copy and never renders a raw provider exception.
- Unknown errors display a generic message plus a trace/support identifier while full diagnostics remain server-side.
- Existing clients using legacy `error`, `errorCode`, and `errorMessage` fields continue to function during migration.
- Automated tests cover security masking, contract compatibility, retry semantics, and user-visible behavior.

View File

@@ -1,104 +0,0 @@
# KnowledgeFS bounded long-PDF parsing
## What changed
- Added process-local single-flight coordination for identical Unstructured requests. Cancellation
removes work that is still waiting for admission, while an already-started transport is allowed
to settle before the caller is released. Heavy-document admission is nested outside the
process-wide gate so a queued heavy request cannot consume ordinary remote-parser capacity.
- Added execution-scoped durable lease protection around remote parser calls. The database attempt
and outbox remain fenced for the parser deadline plus bounded grace, while broker heartbeats stay
at their normal short lease. A result is checkpointed only after the worker reasserts ownership.
Explicit provider responses can release protection; timeout and ambiguous network outcomes do
not launch an overlapping retry.
- Added migration `0047_parse_artifact_checkpoints` for policy-keyed raw Unstructured artifacts.
Completed remote parses can be reused across durable attempts until canonical materialization
succeeds. Native Markdown, HTML, CSV, JSON, XML, and other native parser routes do not read or
write the raw checkpoint table.
- Added a document-scoped Poppler session that materializes the source PDF once, reuses page-size
probes, and releases rendered page buffers before advancing. Cleanup now preserves both the
primary operation error and any cleanup error. Cancellation removes queued render, session, and
materialization work before it acquires a concurrency slot; already-active work still settles
and cleans up before the caller is released.
- Added an opt-in `knowledge-fs-unstructured` Dify Compose profile so KnowledgeFS can use a
page-parallel Unstructured service without changing the existing `unstructured` profile or
legacy Dify ETL traffic.
- Pinned the isolated parser to the tested multi-platform image digest
`sha256:0df934a22e4e893cf15e7aeaf35c463ecc75937758a83099aefdc13041619a1d` and capped it
at four CPUs and 6 GiB.
- Kept ordinary remote parser requests at two concurrent requests with a 600-second (10-minute)
timeout. PDFs and other structurally heavy documents use a nested one-request gate and a
2,400-second timeout.
- Added a tracked required defaults file for service-side split/thread settings, followed by an
optional operator override file. Clean checkouts can therefore run Compose validation without
generating an ignored required file.
- Kept standalone developer-harness host and container endpoints separate:
`http://127.0.0.1:8000` for a source-run API and `http://unstructured:8000` for the API container.
## Representative measurements
These are representative samples, not a complete-document ingestion benchmark:
- A 12-page sample (source pages 8–11, 38–41, and 118–121) took `169.644325 s` with one `hi_res`
request and `90.228331 s` with six-page splits and two child threads, a measured reduction of
`46.813%`. The raw JSON was byte-for-byte identical: 124 elements, all 12 pages, 12 tables with
HTML, and coordinates on all 124 elements.
- An 18-page sample (source pages 8–13, 38–43, and 118–123) took `143.516036 s` with two threads and
`116.074878 s` with three threads, a measured reduction of `19.121%`. The raw JSON was
byte-for-byte identical: 194 elements, all 18 pages, 12 tables, and coordinates on all 194
elements.
- The three-thread run peaked at `390.36%` CPU and `2.916 GiB` parser RSS on a host using
`7.632 GiB`. Those measurements support the one-outer-request, four-CPU, 6-GiB deployment bound;
they do not prove behavior under arbitrary concurrent workloads.
- A Poppler session benchmark used five representative pages (1, 105, 150, 230, and 295) across
six alternating rounds. Median wall time was `1058.096 ms` before and `1050.725 ms` after. The
source PDF was materialized once instead of five times (`3,352,759` versus `16,763,795` bytes),
while output bytes remained identical at `1,352,369`.
The complete approximately 295-page report was **not** run end to end after these changes. A
`31.706 min` full-document duration is only a conservative linear projection from the sample, not
an observed result. The 40-minute PDF timeout therefore represents an estimated `8.294 min` of
headroom and must not be reported as measured end-to-end performance.
## Compatibility and operations
- The legacy `unstructured` service definition, image tag, profile, volume, and behavior are
unchanged.
- The isolated service is internal-only at `http://knowledge_fs_unstructured:8000`; no host port is
published.
- An external Unstructured-compatible endpoint remains supported through `UNSTRUCTURED_API_URL`.
- The upstream synchronous API has no cancellation or idempotency contract. A timed-out request can
leave remote child work running, so Unstructured transport timeouts are not automatically
retried.
- Kubernetes does not receive an implicit parser deployment. Its baseline retains generic limits
unless operators deploy and benchmark an equivalent isolated parser.
- Existing databases must apply migration `0047_parse_artifact_checkpoints` before the new API
image starts. Roll back the application before making any schema change. The checkpoint table is
derived retry state and may be left in place safely; if an operator needs a schema rollback, it
can be dropped after every newer API instance has stopped without affecting canonical parse
artifacts or published document revisions.
- Raw checkpoints are deleted after successful canonical materialization. A terminal downstream
failure keeps one checkpoint per asset/version for a later manual retry; there is currently no
TTL or background checkpoint collector. Document deletion cascades and version pruning reclaim
these rows, and operators should monitor `parse_artifact_checkpoints` growth until a retention
collector is added.
For the bundled Dify deployment, copy the KnowledgeFS service example and start the dedicated
profile explicitly:
```sh
cd docker
cp envs/core-services/knowledge-fs.env.example envs/core-services/knowledge-fs.env
docker compose --profile knowledge-fs-unstructured config
docker compose --profile knowledge-fs-unstructured up -d
```
## Verification
- Dify Compose generation and `knowledge-fs-unstructured` profile configuration.
- Local full-stack and middleware Compose configuration.
- Compose contract tests for isolation, pinned digest, environment precedence, limits, and resource
ceilings.
- KnowledgeFS workflow-path and secret-scan tests.
- Parser coverage and type checks, API/database/adapter test suites, migration artifact validation,
and generated KnowledgeFS contract validation.

1
.nvmrc Normal file
View File

@@ -0,0 +1 @@
22

View File

@@ -1,5 +1,12 @@
#!/bin/sh
set -eu
# get the list of modified files
files=$(git diff --cached --name-only)
# check if api or web directory is modified
api_modified=false
web_modified=false
skip_web_checks=false
git_path() {
git rev-parse --git-path "$1"
@@ -11,7 +18,46 @@ if [ -f "$(git_path MERGE_HEAD)" ] || \
[ -f "$(git_path SQUASH_MSG)" ] || \
[ -d "$(git_path rebase-merge)" ] || \
[ -d "$(git_path rebase-apply)" ]; then
echo "Git operation in progress, skipping staged checks; CI validates the result."
skip_web_checks=true
fi
for file in $files
do
# Use POSIX compliant pattern matching
case "$file" in
api/*.py)
# set api_modified flag to true
api_modified=true
;;
web/*)
# set web_modified flag to true
web_modified=true
;;
esac
done
# run linters based on the modified modules
if $api_modified; then
echo "Running Ruff linter on api module"
# run Ruff linter auto-fixing
uv run --project api --dev ruff check --fix ./api
# run Ruff linter checks
uv run --project api --dev ruff check ./api || status=$?
status=${status:-0}
if [ $status -ne 0 ]; then
echo "Ruff linter on api module error, exit code: $status"
echo "Please run 'dev/reformat' to fix the fixable linting errors."
exit 1
fi
fi
if $skip_web_checks; then
echo "Git operation in progress, skipping web checks"
exit 0
fi

View File

@@ -4,6 +4,7 @@
"tailwindCSS.experimental.configFile": "web/app/styles/globals.css",
// Format
"[javascript]": { "editor.defaultFormatter": "oxc.oxc-vscode" },
"[javascriptreact]": { "editor.defaultFormatter": "oxc.oxc-vscode" },
"[typescript]": { "editor.defaultFormatter": "oxc.oxc-vscode" },
@@ -24,9 +25,9 @@
"[handlebars]": { "editor.defaultFormatter": "oxc.oxc-vscode" },
"editor.formatOnSave": true,
"editor.formatOnSaveMode": "file",
"oxc.disableNestedConfig": true,
"oxc.fmt.disableNestedConfig": true,
"oxc.fmt.configPath": "./vite.config.ts",
// Lint fix
"eslint.useFlatConfig": true,
"eslint.validate": ["json", "jsonc", "markdown", "yaml", "toml"],
"editor.codeActionsOnSave": {

View File

@@ -1,6 +1,6 @@
# AGENTS.md
Dify is an open-source platform for building LLM applications, agentic workflows, and RAG pipelines. This monorepo contains the backend API (`api/`), frontend application (`web/`), deployment assets (`docker/`), standalone agent backend (`dify-agent/`), CLI (`cli/`), and end-to-end suite (`e2e/`). Follow the nearest scoped `AGENTS.md` for the files being changed. Apply its guidance within the user's requested scope; explicit user instructions take precedence over workflow defaults.
Dify is an open-source platform for building LLM applications, agentic workflows, and RAG pipelines. This monorepo contains the backend API (`api/`), frontend application (`web/`), deployment assets (`docker/`), standalone agent backend (`dify-agent/`), CLI (`cli/`), and end-to-end suite (`e2e/`). Follow the nearest scoped `AGENTS.md` for the files being changed.
## Repository Gotchas

View File

@@ -129,35 +129,83 @@ All of Dify's offerings come with corresponding APIs, so you could effortlessly
Star Dify on GitHub and be instantly notified of new releases.
<img width="1344" height="720" alt="star" src="https://github.com/user-attachments/assets/dcd086d1-af0f-471b-ae52-1ad2fa040595" />
![star-us](https://github.com/langgenius/dify/assets/13230914/b823edc1-6388-4e25-ad45-2f6b187adbb4)
## Advanced Setup
For custom configuration, observability, and deployment options, see [Advanced Setup](docs/ADVANCED_SETUP.md).
### Custom configurations
If you need to customize the configuration, edit `docker/.env`. The essential startup defaults live in [`docker/.env.example`](docker/.env.example), and optional advanced variables are split under `docker/envs/` by theme. After making any changes, re-run `docker compose up -d` from the `docker` directory. You can find the full list of available environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
### Metrics Monitoring with Grafana
Import the dashboard to Grafana, using Dify's PostgreSQL database as data source, to monitor metrics in granularity of apps, tenants, messages, and more.
- [Grafana Dashboard by @bowenliang123](https://github.com/bowenliang123/dify-grafana-dashboard)
### Deployment with Kubernetes
If you'd like to configure a highly available setup, there are community-contributed [Helm Charts](https://helm.sh/) and YAML files which allow Dify to be deployed on Kubernetes.
- [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
- [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
- [Helm Chart by @magicsong](https://github.com/magicsong/ai-charts)
- [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
- [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s)
- [🚀 NEW! YAML files (Supports Dify v1.6.0) by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes)
#### Using Terraform for Deployment
Deploy Dify to Cloud Platform with a single click using [terraform](https://www.terraform.io/)
##### Azure Global
- [Azure Terraform by @nikawang](https://github.com/nikawang/dify-azure-terraform)
##### Google Cloud
- [Google Cloud Terraform by @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
#### Using AWS CDK for Deployment
Deploy Dify to AWS with [CDK](https://aws.amazon.com/cdk/)
##### AWS
- [AWS CDK by @KevinZhao (EKS based)](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
- [AWS CDK by @tmokmss (ECS based)](https://github.com/aws-samples/dify-self-hosted-on-aws)
#### Using Alibaba Cloud Computing Nest
Quickly deploy Dify to Alibaba cloud with [Alibaba Cloud Computing Nest](https://computenest.console.aliyun.com/service/instance/create/default?type=user&ServiceName=Dify%E7%A4%BE%E5%8C%BA%E7%89%88)
#### Using Alibaba Cloud Data Management
One-Click deploy Dify to Alibaba Cloud with [Alibaba Cloud Data Management](https://www.alibabacloud.com/help/en/dms/dify-in-invitational-preview/)
#### Deploy to AKS with Azure Devops Pipeline
One-Click deploy Dify to AKS with [Azure Devops Pipeline Helm Chart by @LeoZhang](https://github.com/Ruiruiz30/Dify-helm-chart-AKS)
## Contributing
Dify welcomes contributions of all kinds:
For those who'd like to contribute code, see our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
At the same time, please consider supporting Dify by sharing it on social media and at events and conferences.
- **Code**: Read the [Contribution Guide](CONTRIBUTING.md), then browse [good first issues](https://github.com/langgenius/dify/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22good%20first%20issue%22).
- **Ideas and feedback**: Start or join a [GitHub Discussion](https://github.com/langgenius/dify/discussions).
- **Translations**: Follow the [internationalization guide](web/i18n-config/README.md) to add or update a locale.
- **Community**: Share the apps you build, help other users, and spread the word about Dify.
### Contributors
<a href="https://github.com/langgenius/dify/graphs/contributors">
<img alt="Dify contributors" src="https://contrib.rocks/image?repo=langgenius/dify" />
</a>
> We are looking for contributors to help translate Dify into languages other than Mandarin or English. If you are interested in helping, please see the [i18n README](https://github.com/langgenius/dify/blob/main/web/i18n-config/README.md) for more information, and leave us a comment in the `global-users` channel of our [Discord Community Server](https://discord.gg/8Tpq4AcN9c).
## Community & contact
Choose the channel that best fits your question:
- [GitHub Discussion](https://github.com/langgenius/dify/discussions). Best for: sharing feedback and asking questions.
- [GitHub Issues](https://github.com/langgenius/dify/issues). Best for: bugs you encounter using Dify.AI, and feature proposals. See our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
- [Discord](https://discord.gg/FngNHpbcY7). Best for: sharing your applications and hanging out with the community.
- [X(Twitter)](https://twitter.com/dify_ai). Best for: sharing your applications and hanging out with the community.
- [GitHub Discussions](https://github.com/langgenius/dify/discussions): Get help, share feedback, and propose ideas.
- [GitHub Issues](https://github.com/langgenius/dify/issues): Report reproducible bugs and track engineering work. Read the [Contribution Guide](CONTRIBUTING.md) before opening one.
- [Discord](https://discord.gg/FngNHpbcY7): Chat in real time, share your apps, and connect with other Dify users.
- [X](https://x.com/dify_ai): Follow Dify for release news and project updates.
**Contributors**
<a href="https://github.com/langgenius/dify/graphs/contributors">
<img src="https://contrib.rocks/image?repo=langgenius/dify" />
</a>
## Star History

View File

@@ -98,11 +98,9 @@ REDIS_KEEPALIVE=true
CELERY_BROKER_URL=redis://:difyai123456@localhost:${REDIS_PORT}/1
CELERY_BACKEND=redis
# Ops trace configuration
OPS_TRACE_UNIFIED_ENABLED=false
OPS_TRACE_RETRYABLE_DISPATCH_MAX_RETRIES=780
# Ops trace retry configuration
OPS_TRACE_RETRYABLE_DISPATCH_MAX_RETRIES=60
OPS_TRACE_RETRYABLE_DISPATCH_DELAY_SECONDS=5
OPS_TRACE_PARENT_CONTEXT_TTL_SECONDS=3900
# Database configuration
DB_TYPE=postgresql
@@ -222,7 +220,6 @@ SUPABASE_URL=your-server-url
# CORS configuration
WEB_API_CORS_ALLOW_ORIGINS=http://localhost:3000,*
CONSOLE_CORS_ALLOW_ORIGINS=http://localhost:3000,*
WEBSOCKET_MAX_HTTP_BUFFER_SIZE=10485760
# When the frontend and backend run on different subdomains, set COOKIE_DOMAIN to the site’s top-level domain (e.g., `example.com`). Leading dots are optional.
COOKIE_DOMAIN=
@@ -234,7 +231,6 @@ VECTOR_INDEX_NAME_PREFIX=Vector_index
# Weaviate configuration
WEAVIATE_ENDPOINT=http://localhost:8080
WEAVIATE_GRPC_ENDPOINT=grpc://localhost:50051
WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih
WEAVIATE_BATCH_SIZE=100
WEAVIATE_TOKENIZATION=word
@@ -492,13 +488,6 @@ SENDGRID_API_KEY=
# Sentry configuration
SENTRY_DSN=
# Cloudflare Turnstile server-side verification for Dify Cloud sign-in
TURNSTILE_SECRET_KEY=
# Comma-separated parent or exact hostnames, for example: dify.ai,staging.dify.dev
TURNSTILE_ALLOWED_HOSTNAMES=
# Enable only after the compatible web client has been fully deployed.
TURNSTILE_EMAIL_CODE_VERIFY_REQUIRED=false
# DEBUG
DEBUG=false
ENABLE_REQUEST_LOGGING=False
@@ -589,7 +578,7 @@ INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000
# Workflow runtime configuration
WORKFLOW_MAX_EXECUTION_STEPS=500
WORKFLOW_MAX_EXECUTION_TIME=3600
WORKFLOW_MAX_EXECUTION_TIME=1200
WORKFLOW_CALL_MAX_DEPTH=5
MAX_VARIABLE_SIZE=204800
# Maximum concurrent node-builder LLM calls per workflow generation request
@@ -634,7 +623,7 @@ WORKFLOW_LOG_CLEANUP_BATCH_SIZE=100
WORKFLOW_LOG_CLEANUP_SPECIFIC_WORKFLOW_IDS=
# App configuration
APP_MAX_EXECUTION_TIME=3600
APP_MAX_EXECUTION_TIME=1200
APP_DEFAULT_ACTIVE_REQUESTS=0
APP_MAX_ACTIVE_REQUESTS=0
@@ -709,36 +698,19 @@ INNER_API_KEY_FOR_PLUGIN=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y
# Dify Agent backend
AGENT_BACKEND_BASE_URL=http://localhost:5050
# Bearer token sent to the Agent backend control-plane API. Must match DIFY_AGENT_API_TOKEN on the server side.
# Bearer token sent to the Agent backend /runs API. Must match DIFY_AGENT_API_TOKEN on the server side.
AGENT_BACKEND_API_TOKEN=dify-agent-run-token-for-dev-only
AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS=30
AGENT_BACKEND_STREAM_MAX_RECONNECTS=3
# Client deadline for converting a Binding file to a ToolFile through the Agent backend.
AGENT_BACKEND_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS=240
AGENT_BACKEND_RUN_TIMEOUT_SECONDS=1200
# KnowledgeFS (Dataset 2.0)
KNOWLEDGE_FS_ENABLED=false
# Explicitly bypass the Community edition gate for development deployments.
KNOWLEDGE_FS_COMMUNITY_DEV_ENABLED=false
# Production deployments require HTTPS; plain HTTP is limited to non-production or loopback.
KNOWLEDGE_FS_BASE_URL=
KNOWLEDGE_FS_DIRECT_ORIGIN=
KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED=false
KNOWLEDGE_FS_INTEGRATED_PROVISION_READY=false
KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY=false
KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS=15
KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS=60
KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE=25
KNOWLEDGE_FS_CAPABILITY_V2_ENABLED=false
KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID=
KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM=
KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS=
KNOWLEDGE_FS_CAPABILITY_V2_ISSUER=dify-control-plane
KNOWLEDGE_FS_CAPABILITY_V2_AUDIENCE=knowledge-fs
KNOWLEDGE_FS_CAPABILITY_V2_MAX_TTL_SECONDS=60
# Shared with KnowledgeFS; use at least 32 random characters.
KNOWLEDGE_FS_JWT_SECRET=
KNOWLEDGE_FS_SSE_READ_TIMEOUT_SECONDS=300
KNOWLEDGE_FS_TIMEOUT_SECONDS=10
KNOWLEDGE_FS_JWKS_CACHE_MAX_AGE_SECONDS=300
KNOWLEDGE_FS_PRODUCT_MAX_RESPONSE_BYTES=4194304
# Marketplace configuration
MARKETPLACE_ENABLED=true
@@ -757,8 +729,6 @@ RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5
EMAIL_REGISTER_TOKEN_EXPIRY_MINUTES=5
CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES=5
OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES=5
EMAIL_CODE_LOGIN_TOKEN_EXPIRY_MINUTES=5
EMAIL_CODE_LOGIN_MAX_ATTEMPTS=5
CREATE_TIDB_SERVICE_JOB_ENABLED=false
@@ -855,12 +825,6 @@ ENABLE_HUMAN_INPUT_TIMEOUT_TASK=true
# Human input timeout check interval in minutes
HUMAN_INPUT_TIMEOUT_TASK_INTERVAL=1
# Whether to recover soft-deleted conversation cleanup jobs periodically
ENABLE_CONVERSATION_CLEANUP_TASK=true
# Recovery interval in minutes and maximum conversations dispatched per sweep
CONVERSATION_CLEANUP_TASK_INTERVAL=5
CONVERSATION_CLEANUP_BATCH_SIZE=100
# Nacos remote settings source HTTP timeouts (seconds).
# Bound how long requests to the Nacos endpoint wait before failing, so a slow or
# unresponsive Nacos server cannot stall API startup or token refresh.
@@ -868,6 +832,3 @@ CONVERSATION_CLEANUP_BATCH_SIZE=100
DIFY_ENV_NACOS_REQUEST_TIMEOUT=10.0
# Connect timeout for Nacos requests (default: 3.0)
DIFY_ENV_NACOS_CONNECT_TIMEOUT=3.0
# skill entry
ENABLE_SKILL=true

View File

@@ -15,112 +15,6 @@ root_packages =
services
include_external_packages = True
[importlinter:contract:backend-layers]
name = Backend modules follow the transport, application, domain, and shared-library layers
type = layers
layers =
controllers
services
core
libs
# Migration baseline: keep every exception exact. The default unmatched-import
# error makes this list shrink whenever a legacy dependency is removed.
unmatched_ignore_imports_alerting = error
ignore_imports =
core.app.app_config.easy_ui_based_app.dataset.manager -> services.dataset_service
core.app.apps.advanced_chat.app_generator -> controllers.console.app.workflow
core.app.apps.advanced_chat.app_generator -> services.conversation_service
core.app.apps.advanced_chat.app_generator -> services.errors.conversation
core.app.apps.advanced_chat.app_generator -> services.workflow_draft_variable_service
core.app.apps.advanced_chat.app_runner -> services.conversation_variable_updater
core.app.apps.agent_app.app_generator -> services.agent.composer_service
core.app.apps.agent_app.app_generator -> services.agent.workspace_service
core.app.apps.agent_app.app_generator -> services.conversation_service
core.app.apps.agent_app.runtime_request_builder -> services.agent.prompt_mentions
core.app.apps.agent_app.session_store -> services.agent.workspace_service
core.app.apps.agent_chat.app_generator -> services.conversation_service
core.app.apps.base_app_generator -> services.workflow_draft_variable_service
core.app.apps.chat.app_generator -> services.conversation_service
core.app.apps.common.workflow_response_converter -> services.variable_truncator
core.app.apps.completion.app_generator -> services.errors.app
core.app.apps.completion.app_generator -> services.errors.message
core.app.apps.message_based_app_generator -> services.errors.app_model_config
core.app.apps.message_based_app_generator -> services.errors.conversation
core.app.apps.message_based_app_generator -> services.errors.message
core.app.apps.pipeline.pipeline_generator -> services
core.app.apps.pipeline.pipeline_generator -> services.dataset_service
core.app.apps.pipeline.pipeline_generator -> services.datasource_provider_service
core.app.apps.pipeline.pipeline_generator -> services.feature_service
core.app.apps.pipeline.pipeline_generator -> services.workflow_draft_variable_service
core.app.apps.pipeline.pipeline_runner -> services.dataset_ref_service
core.app.apps.workflow.app_generator -> controllers.console.app.workflow
core.app.apps.workflow.app_generator -> services.snippet_generate_service
core.app.apps.workflow.app_generator -> services.workflow_draft_variable_service
core.app.apps.workflow_app_runner -> tasks.mail_human_input_delivery_task
core.app.features.annotation_reply.annotation_reply -> services.annotation_service
core.app.features.annotation_reply.annotation_reply -> services.dataset_service
core.app.llm.quota -> services.credit_pool_service
core.app.task_pipeline.message_cycle_manager -> services.account_service
core.app.task_pipeline.message_cycle_manager -> services.annotation_service
core.app.workflow.layers.persistence -> services.workflow.inspector_events
core.datasource.datasource_manager -> services.datasource_provider_service
core.helper.credential_utils -> services.enterprise.plugin_manager_service
core.helper.credential_utils -> services.system_feature_service
core.indexing_runner -> services.vector_space_admission_service
core.mcp.auth_client -> services
core.provider_manager -> services.credential_permission_service
core.provider_manager -> services.credit_pool_service
core.provider_manager -> services.feature_service
core.rag.datasource.retrieval_service -> services.external_knowledge_service
core.rag.index_processor.index_processor -> services.vector_space_admission_service
core.rag.index_processor.index_processor_base -> services.file_service
core.rag.index_processor.processor.paragraph_index_processor -> services.account_service
core.rag.index_processor.processor.paragraph_index_processor -> services.summary_index_service
core.rag.index_processor.processor.parent_child_index_processor -> services.account_service
core.rag.index_processor.processor.parent_child_index_processor -> services.summary_index_service
core.rag.index_processor.processor.qa_index_processor -> services.summary_index_service
core.rag.retrieval.dataset_retrieval -> services.external_knowledge_service
core.rag.retrieval.dataset_retrieval -> services.feature_service
core.rag.summary_index.summary_index -> services.summary_index_service
core.rag.summary_index.summary_index -> tasks.generate_summary_index_task
core.repositories.sqlalchemy_workflow_node_execution_repository -> services.file_service
core.repositories.sqlalchemy_workflow_node_execution_repository -> services.variable_truncator
core.tools.tool_manager -> services
core.workflow.node_runtime -> services
core.workflow.nodes.agent.message_transformer -> services
core.workflow.nodes.agent_v2.agent_node -> services.agent.prompt_mentions
core.workflow.nodes.agent_v2.agent_node -> services.agent.workspace_service
core.workflow.nodes.agent_v2.dify_tools_builder -> services
core.workflow.nodes.agent_v2.runtime_feature_manifest -> services.agent.knowledge_datasets
core.workflow.nodes.agent_v2.runtime_request_builder -> services.agent.prompt_mentions
core.workflow.nodes.agent_v2.runtime_request_builder -> services.skill_management_service
core.workflow.nodes.agent_v2.session_store -> services.agent.workspace_service
core.workflow.nodes.agent_v2.validators -> services.agent.knowledge_datasets
core.workflow.nodes.agent_v2.workspace_retirement_layer -> tasks.collect_agent_resources_task
libs.device_flow_security -> controllers.openapi._models
libs.device_flow_security -> services.entities.feature_entities
libs.device_flow_security -> services.system_feature_service
libs.email_i18n -> services.entities.feature_entities
libs.email_i18n -> services.system_feature_service
libs.external_api -> core
libs.external_api -> core.errors.error
libs.external_api -> extensions.ext_logging
libs.flask_utils -> models
libs.helper -> core.app.features.rate_limiting.rate_limit
libs.helper -> models
libs.helper -> models.model
libs.json_in_md_parser -> core.llm_generator.output_parser.errors
libs.login -> extensions.ext_login
libs.login -> models
libs.login -> models.model
libs.oauth -> core.helper.http_client_pooling
libs.oauth_bearer -> models
libs.rsa -> extensions.ext_storage
libs.workspace_permission -> services.enterprise.enterprise_service
services.account_service -> controllers
services.account_service -> controllers.console.error
services.app_generate_service -> controllers.console.app.workflow
[importlinter:contract:no-direct-rsa-imports]
# Note: `libs` itself is deliberately excluded from source_modules -- import-linter's
# `forbidden` contract cannot check a package against its own descendant (libs.rsa lives
@@ -171,177 +65,6 @@ forbidden_modules =
sqlalchemy
werkzeug
[importlinter:contract:init-validation-service-boundary]
name = Initialization validation application service is framework and persistence neutral
type = forbidden
source_modules =
services.init_validation_service
forbidden_modules =
controllers
extensions
flask
models
repositories
sqlalchemy
werkzeug
[importlinter:contract:explore-banner-query-service-boundary]
name = Explore banner query application service is framework and persistence neutral
type = forbidden
source_modules =
services.explore_banner_query_service
forbidden_modules =
controllers
extensions
flask
models
repositories
sqlalchemy
werkzeug
[importlinter:contract:account-application-boundary]
name = Account application services and contracts are framework and persistence neutral
type = forbidden
source_modules =
services.account_access_service
services.account_avatar_service
services.account_change_email_ports
services.account_change_email_service
services.account_email_registration_service
services.account_deletion_service
services.account_deletion_feedback_service
services.account_education_service
services.account_errors
services.account_forgot_password_service
services.account_initialization_service
services.account_integration_service
services.account_login_service
services.account_oauth_service
services.account_password_service
services.account_ports
services.account_profile_service
services.entities.account_access_entities
services.entities.account_entities
services.entities.account_login_entities
services.entities.account_oauth_entities
services.entities.auth_audit_entities
forbidden_modules =
configs
controllers
extensions
flask
models
repositories
sqlalchemy
werkzeug
[importlinter:contract:app-definition-query-service-boundary]
name = App definition query application service is framework and persistence neutral
type = forbidden
source_modules =
services.app_definition_query_service
forbidden_modules =
controllers
extensions
flask
models
repositories
sqlalchemy
werkzeug
[importlinter:contract:app-site-service-boundary]
name = App site application service is framework and persistence neutral
type = forbidden
source_modules =
services.app_site_service
forbidden_modules =
configs
controllers
extensions
flask
models
repositories
sqlalchemy
werkzeug
[importlinter:contract:app-tracing-config-service-boundary]
name = App tracing configuration service is framework and persistence neutral
type = forbidden
source_modules =
services.app_tracing_config_service
forbidden_modules =
configs
controllers
extensions
flask
models
repositories
sqlalchemy
werkzeug
[importlinter:contract:message-file-preview-service-boundary]
name = Message file preview application service is framework and persistence neutral
type = forbidden
source_modules =
services.message_file_preview_service
forbidden_modules =
controllers
extensions
flask
models
repositories
sqlalchemy
werkzeug
[importlinter:contract:upload-file-delivery-service-boundary]
name = Upload file delivery application service is framework and persistence neutral
type = forbidden
source_modules =
services.upload_file_delivery_service
forbidden_modules =
controllers
extensions
flask
models
repositories
sqlalchemy
werkzeug
[importlinter:contract:webapp-access-query-service-boundary]
name = Web app access query application service is framework and persistence neutral
type = forbidden
source_modules =
services.webapp_access_query_service
forbidden_modules =
configs
controllers
extensions
flask
models
repositories
services.enterprise
services.feature_service
sqlalchemy
werkzeug
[importlinter:contract:web-app-runtime-query-service-boundary]
name = Web app runtime query application service does not directly depend on configuration, feature implementation, transport, or ORM modules
type = forbidden
source_modules =
services.web_app_runtime_query_service
forbidden_modules =
configs
controllers
extensions
flask
models
repositories
services.feature_service
services.feature_service_gateway
sqlalchemy
werkzeug
allow_indirect_imports = True
[importlinter:contract:feature-query-service-boundary]
name = Feature query application service is framework and persistence neutral
type = forbidden
@@ -356,40 +79,6 @@ forbidden_modules =
sqlalchemy
werkzeug
[importlinter:contract:data-source-oauth-service-boundary]
name = Data source OAuth application service is framework and persistence neutral
type = forbidden
source_modules =
services.data_source_oauth_service
forbidden_modules =
configs
controllers
extensions
flask
httpx
libs
models
repositories
sqlalchemy
werkzeug
[importlinter:contract:oauth-server-application-service-boundary]
name = OAuth server application service is framework and persistence neutral
type = forbidden
source_modules =
services.oauth_server_service
forbidden_modules =
configs
controllers
extensions
flask
httpx
libs
models
repositories
sqlalchemy
werkzeug
[importlinter:contract:workspace-member-query-service-boundary]
name = Workspace member query application service is framework and persistence neutral
type = forbidden
@@ -436,169 +125,3 @@ forbidden_modules =
repositories
sqlalchemy
werkzeug
[importlinter:contract:inner-mail-service-boundary]
name = Inner mail application service is framework and implementation neutral
type = forbidden
source_modules =
services.inner_mail_service
forbidden_modules =
configs
controllers
extensions
flask
models
repositories
sqlalchemy
tasks
werkzeug
[importlinter:contract:web-passport-service-boundary]
name = Web passport application service is framework and persistence neutral
type = forbidden
source_modules =
services.web_passport_service
forbidden_modules =
configs
controllers
extensions
flask
models
repositories
sqlalchemy
werkzeug
[importlinter:contract:file-grant-service-boundary]
name = File grant application service is framework and persistence neutral
type = forbidden
source_modules =
services.file_grant_service
services.entities.file_grant_entities
forbidden_modules =
configs
controllers
extensions
flask
models
repositories
sqlalchemy
werkzeug
[importlinter:contract:plugin-file-upload-service-boundary]
name = Plugin file upload application service is framework and persistence neutral
type = forbidden
source_modules =
services.plugin_file_upload_service
forbidden_modules =
controllers
extensions
flask
models
repositories
sqlalchemy
werkzeug
[importlinter:contract:account-activation-service-boundary]
name = Account activation application service is framework and persistence neutral
type = forbidden
source_modules =
services.account_activation_service
services.entities.account_activation_entities
forbidden_modules =
configs
controllers
extensions
flask
models
repositories
sqlalchemy
werkzeug
[importlinter:contract:tag-application-service-boundary]
name = Tag application service is framework and persistence neutral
type = forbidden
source_modules =
services.tag_application_service
forbidden_modules =
configs
controllers
extensions
flask
models
repositories
sqlalchemy
werkzeug
[importlinter:contract:tool-file-download-service-boundary]
name = Tool file download application service is framework and persistence neutral
type = forbidden
source_modules =
services.tool_file_download_service
forbidden_modules =
controllers
extensions
flask
models
repositories
sqlalchemy
werkzeug
[importlinter:contract:recommended-app-query-service-boundary]
name = Recommended app query application service is framework and persistence neutral
type = forbidden
source_modules =
services.recommended_app_query_service
forbidden_modules =
configs
controllers
extensions
flask
models
repositories
services.feature_service
services.recommended_app_catalog_gateway
sqlalchemy
werkzeug
[importlinter:contract:recommended-app-catalog-gateway-boundary]
name = Recommended app catalog gateway does not depend on Flask
type = forbidden
source_modules =
services.recommended_app_catalog_gateway
forbidden_modules =
flask
[importlinter:contract:billing-application-boundary]
name = Billing application services and ports are framework and infrastructure neutral
type = forbidden
source_modules =
services.billing_portal_service
services.compliance_download_service
services.partner_tenant_binding_service
forbidden_modules =
configs
controllers
extensions
flask
models
repositories
services.billing_service
sqlalchemy
werkzeug
[importlinter:contract:workflow-run-archive-application-boundary]
name = Workflow run archive application service and task domain are framework and persistence neutral
type = forbidden
source_modules =
services.retention.workflow_run.archive_download_task
services.retention.workflow_run.archive_log_service
forbidden_modules =
configs
controllers
extensions
flask
models
repositories
sqlalchemy
tasks
werkzeug

View File

@@ -1,19 +1,3 @@
# The same API image can serve HTTP or consume KnowledgeFS Celery queues. Bundle the existing
# TypeScript engine at build time; workers must not download code or call an API Pod to execute it.
FROM node:22-bookworm-slim AS knowledge-engine
ENV PNPM_HOME=/pnpm
ENV PATH="${PNPM_HOME}:${PATH}"
WORKDIR /knowledge-fs
RUN corepack enable
COPY knowledge-fs/package.json knowledge-fs/pnpm-lock.yaml knowledge-fs/pnpm-workspace.yaml knowledge-fs/turbo.json knowledge-fs/tsconfig.base.json ./
COPY knowledge-fs/apps/api ./apps/api
COPY knowledge-fs/packages ./packages
RUN pnpm install --frozen-lockfile --filter @knowledge/api-app... \
&& pnpm --filter @knowledge/api-app build:prod
RUN sharp_runtime_source="$(dirname "$(realpath apps/api/node_modules/sharp)")" \
&& mkdir -p /engine-node-modules \
&& cp -LR "${sharp_runtime_source}/." /engine-node-modules/
# base image
FROM python:3.12-slim-bookworm AS base
@@ -49,6 +33,7 @@ RUN uv sync --frozen --no-dev --no-editable
FROM base AS production
ENV FLASK_APP=app.py
ENV EDITION=SELF_HOSTED
ENV DEPLOY_ENV=PRODUCTION
ENV CONSOLE_API_URL=http://127.0.0.1:5001
ENV CONSOLE_WEB_URL=http://127.0.0.1:3000
@@ -69,14 +54,15 @@ WORKDIR /app/api
# Create non-root user
ARG dify_uid=1001
ARG NODE_PACKAGE_VERSION=24.20.0-1nodesource1
ARG NODE_MAJOR=22
ARG NODE_PACKAGE_VERSION=22.21.0-1nodesource1
ARG NODESOURCE_KEY_FPR=6F71F525282841EEDAF851B42F59B5F99B1BE0B4
RUN groupadd -r -g ${dify_uid} dify && \
useradd -r -u ${dify_uid} -g ${dify_uid} -s /bin/bash dify && \
chown -R dify:dify /app
RUN NODE_MAJOR="${NODE_PACKAGE_VERSION%%.*}" \
&& apt-get update \
RUN \
apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
curl \
@@ -105,8 +91,6 @@ RUN NODE_MAJOR="${NODE_PACKAGE_VERSION%%.*}" \
media-types \
# install libmagic to support the use of python-magic guess MIMETYPE
libmagic1 \
# Bounded PDF metadata/rasterization invoked by the local KnowledgeFS worker engine.
poppler-utils \
&& apt-get autoremove -y \
&& rm -rf /var/lib/apt/lists/*
@@ -127,10 +111,6 @@ RUN python -c "import tiktoken; tiktoken.encoding_for_model('gpt2')" \
# Copy source code
COPY --chown=dify:dify api /app/api/
COPY --from=knowledge-engine --chown=dify:dify /knowledge-fs/apps/api/dist/celery-worker.mjs /app/knowledge-fs/celery-worker.mjs
COPY --from=knowledge-engine --chown=dify:dify /knowledge-fs/apps/api/dist/native-parser-worker.mjs /app/knowledge-fs/native-parser-worker.mjs
COPY --from=knowledge-engine --chown=dify:dify /knowledge-fs/apps/api/dist/image-variant-worker.mjs /app/knowledge-fs/image-variant-worker.mjs
COPY --from=knowledge-engine --chown=dify:dify /engine-node-modules /app/knowledge-fs/node_modules
# Prepare entrypoint script
COPY --chown=dify:dify --chmod=755 api/docker/entrypoint.sh /entrypoint.sh

View File

@@ -7,27 +7,10 @@
!dify-agent/README.md
!dify-agent/src/
!dify-agent/src/**
!knowledge-fs/
!knowledge-fs/package.json
!knowledge-fs/pnpm-lock.yaml
!knowledge-fs/pnpm-workspace.yaml
!knowledge-fs/turbo.json
!knowledge-fs/tsconfig.base.json
!knowledge-fs/apps/
!knowledge-fs/apps/api/
!knowledge-fs/apps/api/**
!knowledge-fs/packages/
!knowledge-fs/packages/**
**/node_modules/
**/dist/
**/.turbo/
# Environment configuration and example
.env
*.env.*
**/.env
**/*.env
**/*.env.*
# Python related files
**/__pycache__

View File

@@ -50,9 +50,7 @@ The scripts resolve paths relative to their location, so you can run them from a
./dev/start-worker
```
1. Start Celery Beat when scheduled tasks are needed. This is required when
`KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED=true` so provisioning and deletion
outbox commands are dispatched to the lifecycle worker.
1. Optional: start Celery Beat (scheduled tasks).
```bash
./dev/start-beat
@@ -91,15 +89,14 @@ The scripts resolve paths relative to their location, so you can run them from a
1. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml`, more can check [Claude.md](../CLAUDE.md)
Continue in the `api` directory from the previous step.
```bash
cd api
uv run pytest # Run all tests
uv run pytest tests/unit_tests/ # Unit tests only
uv run pytest tests/integration_tests/ # Integration tests
# Code quality
../dev/reformat # Run all formatters and linters
./dev/reformat # Run all formatters and linters
uv run ruff check --fix ./ # Fix linting issues
uv run ruff format ./ # Format code
uv run pyrefly check # Type checking

View File

@@ -14,7 +14,6 @@ from contexts.wrapper import RecyclableContextVar
from controllers.console.error import UnauthorizedAndForceLogout
from core.logging.context import init_request_context
from dify_app import DifyApp
from enums import DeploymentEdition
from extensions.ext_socketio import sio
from services.enterprise.enterprise_service import EnterpriseService
from services.entities.feature_entities import LicenseStatus
@@ -105,10 +104,6 @@ def create_flask_app_with_configs() -> DifyApp:
dify_app = DifyApp(__name__)
dify_app.config.from_mapping(dify_config.model_dump())
dify_app.config["RESTX_INCLUDE_ALL_MODELS"] = True
# flask-restx appends url-map suggestions to 404 bodies: they enumerate routes to
# anonymous callers, and a policy 404 on an existing route (edition admission)
# ends up suggesting the very path that was just requested.
dify_app.config["RESTX_ERROR_404_HELP"] = False
# add before request hook
@dify_app.before_request
@@ -117,7 +112,7 @@ def create_flask_app_with_configs() -> DifyApp:
init_request_context()
RecyclableContextVar.increment_thread_recycles()
if dify_config.DEPLOYMENT_EDITION == DeploymentEdition.ENTERPRISE:
if dify_config.ENTERPRISE_ENABLED:
surface = _match_license_gated_surface(request.path)
if surface is not None:
try:
@@ -190,7 +185,6 @@ def initialize_extensions(app: DifyApp):
ext_hosting_provider,
ext_import_modules,
ext_key_provider,
ext_knowledge_fs_observability,
ext_logging,
ext_login,
ext_logstore,
@@ -242,7 +236,6 @@ def initialize_extensions(app: DifyApp):
ext_enterprise_telemetry,
ext_request_logging,
ext_session_factory,
ext_knowledge_fs_observability,
ext_application_services,
ext_oauth_bearer,
]

View File

@@ -1,15 +1,11 @@
from gevent.monkey import is_module_patched
import psycogreen.gevent as pscycogreen_gevent
from grpc.experimental import gevent as grpc_gevent
# Celery patches the stdlib before importing this entrypoint for -P gevent. Prefork workers
# (including the KnowledgeFS local engine) must retain native subprocess, DB and gRPC behavior.
if is_module_patched("socket"):
import psycogreen.gevent as pscycogreen_gevent
from grpc.experimental import gevent as grpc_gevent
grpc_gevent.init_gevent()
print("gRPC patched with gevent.", flush=True) # noqa: T201
pscycogreen_gevent.patch_psycopg()
print("psycopg2 patched with gevent.", flush=True) # noqa: T201
# grpc gevent
grpc_gevent.init_gevent()
print("gRPC patched with gevent.", flush=True) # noqa: T201
pscycogreen_gevent.patch_psycopg()
print("psycopg2 patched with gevent.", flush=True) # noqa: T201
from app import app, celery

View File

@@ -23,7 +23,6 @@ from dify_agent.protocol import (
CancelRunResponse,
CreateRunRequest,
CreateRunResponse,
RunCancelledEvent,
RunEvent,
RunStatusResponse,
)
@@ -46,15 +45,6 @@ class AgentBackendRunClient(Protocol):
def cancel_run(self, run_id: str, request: CancelRunRequest | None = None) -> CancelRunResponse:
"""Request explicit cancellation for one Agent backend run."""
def cancel_run_and_wait(
self,
run_id: str,
request: CancelRunRequest | None = None,
*,
after: str | None = None,
) -> RunCancelledEvent:
"""Request cancellation and wait for runner cleanup to finish."""
def stream_events(
self,
run_id: str,
@@ -77,15 +67,6 @@ class _DifyAgentSyncClient(Protocol):
def cancel_run_sync(self, run_id: str, request: CancelRunRequest | None = None) -> CancelRunResponse:
"""Cancel one run synchronously."""
def cancel_run_and_wait_sync(
self,
run_id: str,
request: CancelRunRequest | None = None,
*,
after: str | None = None,
) -> RunCancelledEvent:
"""Cancel one run and wait for its terminal event synchronously."""
def stream_events_sync(
self,
run_id: str,
@@ -111,9 +92,11 @@ class DifyAgentBackendRunClient:
client: _DifyAgentSyncClient,
*,
stream_max_reconnects: int = 3,
stream_timeout_seconds: float = 1200,
) -> None:
self.client = client
self._stream_max_reconnects = stream_max_reconnects
self._stream_timeout_seconds = stream_timeout_seconds
def create_run(self, request: CreateRunRequest) -> CreateRunResponse:
"""Create one run through ``POST /runs`` and normalize client exceptions."""
@@ -129,19 +112,6 @@ class DifyAgentBackendRunClient:
except Exception as exc:
raise _normalize_dify_agent_error(exc) from exc
def cancel_run_and_wait(
self,
run_id: str,
request: CancelRunRequest | None = None,
*,
after: str | None = None,
) -> RunCancelledEvent:
"""Cancel one run, then wait for the cleanup-complete terminal event."""
try:
return self.client.cancel_run_and_wait_sync(run_id, request=request, after=after)
except Exception as exc:
raise _normalize_dify_agent_error(exc) from exc
def stream_events(
self,
run_id: str,
@@ -155,6 +125,7 @@ class DifyAgentBackendRunClient:
run_id,
after=after,
max_reconnects=self._stream_max_reconnects,
timeout_seconds=self._stream_timeout_seconds,
should_stop=should_stop,
)
except Exception as exc:

View File

@@ -8,40 +8,11 @@ classes.
from __future__ import annotations
from http import HTTPStatus
from typing import Any
from dify_agent.client import DifyAgentHTTPError
from dify_agent.protocol import RunFailureType
def backend_reported_failure(exc: DifyAgentHTTPError) -> bool:
"""Whether the backend itself reported the failure.
``DifyAgentValidationError`` is a ``DifyAgentHTTPError`` that the client also
raises when a *successful* response fails DTO validation. Those carry a 2xx
status and a Pydantic error list, not a backend-authored error envelope.
"""
return exc.status_code >= HTTPStatus.BAD_REQUEST
def backend_error_detail(
exc: DifyAgentHTTPError,
*,
default_code: str = "agent_backend_error",
) -> tuple[str, str]:
"""Split an Agent backend HTTP error into the code and message the backend itself sent."""
detail = exc.detail
if isinstance(detail, dict):
code = detail.get("code")
message = detail.get("message")
return (
code if isinstance(code, str) and code else default_code,
message if isinstance(message, str) and message else str(exc),
)
return default_code, str(detail)
class AgentBackendError(Exception):
"""Base error for API-side Agent backend integration failures."""

View File

@@ -99,8 +99,6 @@ class AgentBackendRunFailedInternalEvent(AgentBackendInternalEventBase):
error: str
error_type: RunFailureType | None = None
reason: str | None = None
session_snapshot: CompositorSessionSnapshot | None = None
usage: dict[str, JsonValue] | None = None
class AgentBackendRunCancelledInternalEvent(AgentBackendInternalEventBase):
@@ -109,8 +107,6 @@ class AgentBackendRunCancelledInternalEvent(AgentBackendInternalEventBase):
type: Literal[AgentBackendInternalEventType.RUN_CANCELLED] = AgentBackendInternalEventType.RUN_CANCELLED
reason: str | None = None
message: str | None = None
session_snapshot: CompositorSessionSnapshot | None = None
usage: dict[str, JsonValue] | None = None
type AgentBackendInternalEvent = Annotated[
@@ -188,8 +184,6 @@ class AgentBackendRunEventAdapter:
error=event.data.error,
error_type=event.data.error_type,
reason=event.data.reason,
session_snapshot=event.data.session_snapshot,
usage=_agent_run_usage(event.data.usage),
)
]
case RunCancelledEvent():
@@ -199,8 +193,6 @@ class AgentBackendRunEventAdapter:
source_event_id=event.id,
reason=event.data.reason,
message=event.data.message,
session_snapshot=event.data.session_snapshot,
usage=_agent_run_usage(event.data.usage),
)
]
raise TypeError(f"unsupported agent backend run event: {type(event).__name__}")

View File

@@ -8,25 +8,6 @@ from clients.agent_backend.client import AgentBackendRunClient, DifyAgentBackend
from clients.agent_backend.fake_client import FakeAgentBackendRunClient, FakeAgentBackendScenario
def create_agent_backend_client(
*,
base_url: str,
api_token: str | None = None,
stream_timeout: float = 30,
timeout: float = 30.0,
binding_file_download_timeout: float = 240,
) -> Client:
api_token = api_token.strip() if api_token else None
headers = {"Authorization": f"Bearer {api_token}"} if api_token else None
return Client(
base_url=base_url,
stream_timeout=stream_timeout,
timeout=timeout,
binding_file_download_timeout=binding_file_download_timeout,
headers=headers,
)
def create_agent_backend_run_client(
*,
base_url: str | None = None,
@@ -35,17 +16,18 @@ def create_agent_backend_run_client(
fake_scenario: str | FakeAgentBackendScenario = FakeAgentBackendScenario.SUCCESS,
stream_read_timeout_seconds: float = 30,
stream_max_reconnects: int = 3,
stream_run_timeout_seconds: float = 1200,
) -> AgentBackendRunClient:
"""Create the API-side run client without hiding the ``dify-agent`` protocol."""
if use_fake:
return FakeAgentBackendRunClient(scenario=FakeAgentBackendScenario(fake_scenario))
if base_url is None:
raise ValueError("base_url is required when creating a real Agent backend client")
headers: dict[str, str] = {}
if api_token:
headers["Authorization"] = f"Bearer {api_token}"
return DifyAgentBackendRunClient(
create_agent_backend_client(
base_url=base_url,
api_token=api_token,
stream_timeout=stream_read_timeout_seconds,
),
Client(base_url=base_url, stream_timeout=stream_read_timeout_seconds, headers=headers),
stream_max_reconnects=stream_max_reconnects,
stream_timeout_seconds=stream_run_timeout_seconds,
)

View File

@@ -18,8 +18,6 @@ from dify_agent.protocol import (
CreateRunRequest,
CreateRunResponse,
DeferredToolCallPayload,
RunCancelledEvent,
RunCancelledEventData,
RunEvent,
RunFailedEvent,
RunFailedEventData,
@@ -71,28 +69,6 @@ class FakeAgentBackendRunClient:
del request
return CancelRunResponse(run_id=run_id, status="cancelled")
def cancel_run_and_wait(
self,
run_id: str,
request: CancelRunRequest | None = None,
*,
after: str | None = None,
) -> RunCancelledEvent:
"""Return a deterministic cleanup-complete cancellation event."""
del after
request = request or CancelRunRequest()
_ = self.cancel_run(run_id, request)
return RunCancelledEvent(
id="cancel-0",
run_id=run_id,
created_at=_FIXED_TIME,
data=RunCancelledEventData(
reason=request.reason,
message=request.message,
session_snapshot=CompositorSessionSnapshot(layers=[]),
),
)
def stream_events(
self,
run_id: str,
@@ -157,11 +133,7 @@ class FakeAgentBackendRunClient:
id="2-0",
run_id=run_id,
created_at=_FIXED_TIME,
data=RunFailedEventData(
error="fake failure",
reason="unit_test",
session_snapshot=CompositorSessionSnapshot(layers=[]),
),
data=RunFailedEventData(error="fake failure", reason="unit_test"),
),
)
case FakeAgentBackendScenario.PAUSED:

View File

@@ -25,23 +25,19 @@ from dify_agent.layers.dify_core_tools import DIFY_CORE_TOOLS_LAYER_TYPE_ID, Dif
from dify_agent.layers.dify_plugin import (
DIFY_PLUGIN_LLM_LAYER_TYPE_ID,
DIFY_PLUGIN_TOOLS_LAYER_TYPE_ID,
DifyPluginCredentialValue,
DifyPluginLLMLayerConfig,
DifyPluginToolsLayerConfig,
)
from dify_agent.layers.drive import DIFY_DRIVE_LAYER_TYPE_ID, DifyDriveLayerConfig
from dify_agent.layers.execution_context import (
DIFY_EXECUTION_CONTEXT_LAYER_TYPE_ID,
DifyExecutionContextLayerConfig,
)
from dify_agent.layers.knowledge import DIFY_KNOWLEDGE_BASE_LAYER_TYPE_ID, DifyKnowledgeBaseLayerConfig
from dify_agent.layers.knowledge_fs import DIFY_KNOWLEDGE_FS_LAYER_TYPE_ID, DifyKnowledgeFsLayerConfig
from dify_agent.layers.output import DIFY_OUTPUT_LAYER_TYPE_ID, DifyOutputLayerConfig
from dify_agent.layers.runtime import DIFY_RUNTIME_LAYER_TYPE_ID, DifyRuntimeLayerConfig
from dify_agent.layers.shell import DIFY_SHELL_LAYER_TYPE_ID, DifyShellLayerConfig
from dify_agent.layers.user_prompt import (
DIFY_USER_PROMPT_LAYER_TYPE_ID,
DifyUserPromptFileConfig,
DifyUserPromptLayerConfig,
)
from dify_agent.protocol import (
DIFY_AGENT_HISTORY_LAYER_ID,
DIFY_AGENT_MODEL_LAYER_ID,
@@ -61,6 +57,7 @@ AGENT_APP_USER_PROMPT_LAYER_ID = "agent_app_user_prompt"
DIFY_EXECUTION_CONTEXT_LAYER_ID = "execution_context"
DIFY_RUNTIME_LAYER_ID = "runtime"
DIFY_CONFIG_LAYER_ID = "config"
DIFY_DRIVE_LAYER_ID = "drive"
DIFY_PLUGIN_TOOLS_LAYER_ID = "tools"
DIFY_CORE_TOOLS_LAYER_ID = "core_tools"
DIFY_KNOWLEDGE_BASE_LAYER_ID = "knowledge"
@@ -76,19 +73,22 @@ def _shell_layer_deps() -> dict[str, str]:
}
def _drive_layer_deps() -> dict[str, str]:
return {"shell": DIFY_SHELL_LAYER_ID}
def _config_layer_deps() -> dict[str, str]:
return {"shell": DIFY_SHELL_LAYER_ID}
def _knowledge_fs_spec(run_input: AgentBackendWorkflowNodeRunInput | AgentBackendAgentAppRunInput) -> RunLayerSpec:
if not run_input.include_shell or run_input.knowledge is not None:
raise ValueError("KnowledgeFS requires shell and cannot be combined with legacy knowledge retrieval")
return RunLayerSpec(
name="knowledge_fs",
type=DIFY_KNOWLEDGE_FS_LAYER_TYPE_ID,
deps={"execution_context": DIFY_EXECUTION_CONTEXT_LAYER_ID, "shell": DIFY_SHELL_LAYER_ID},
config=run_input.knowledge_fs,
)
def _shell_config_with_drive_ref(
shell_config: DifyShellLayerConfig | None,
drive_config: DifyDriveLayerConfig | None,
) -> DifyShellLayerConfig:
config = shell_config or DifyShellLayerConfig()
if drive_config is None:
return config
return config.model_copy(update={"agent_stub_drive_ref": drive_config.drive_ref})
def _markdown_backtick_fence(text: str) -> str:
@@ -150,8 +150,8 @@ class AgentBackendModelConfig(BaseModel):
plugin_id: str
model_provider: str
model: str
credentials: dict[str, DifyPluginCredentialValue] = Field(default_factory=dict)
model_settings: dict[str, JsonValue] = Field(default_factory=dict)
context_window_tokens: int | None = Field(default=None, gt=0)
model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
@@ -224,8 +224,10 @@ class AgentBackendWorkflowNodeRunInput(BaseModel):
tools: DifyPluginToolsLayerConfig | None = None
core_tools: DifyCoreToolsLayerConfig | None = None
knowledge: DifyKnowledgeBaseLayerConfig | None = None
knowledge_fs: DifyKnowledgeFsLayerConfig | None = None
config_layer_config: DifyConfigLayerConfig | None = None
# Drive Skills & Files declaration (dify.drive) — an index the agent pulls
# through the back proxy, never inline content.
drive_config: DifyDriveLayerConfig | None = None
# Human-in-the-loop ask_human deferred tool (dify.ask_human). Present only when
# the Agent Soul configures human involvement; a deferred call ends the run and
# the workflow pauses via the existing HITL form mechanism (ENG-635).
@@ -264,7 +266,6 @@ class AgentBackendAgentAppRunInput(BaseModel):
execution_context: DifyExecutionContextLayerConfig
backend_binding_ref: str = Field(min_length=1)
user_prompt: str
user_files: list[DifyUserPromptFileConfig] = Field(default_factory=list)
agent_soul_prompt: str | None = None
agent_config_version_kind: AgentConfigVersionKind = "snapshot"
idempotency_key: str | None = None
@@ -272,8 +273,10 @@ class AgentBackendAgentAppRunInput(BaseModel):
tools: DifyPluginToolsLayerConfig | None = None
core_tools: DifyCoreToolsLayerConfig | None = None
knowledge: DifyKnowledgeBaseLayerConfig | None = None
knowledge_fs: DifyKnowledgeFsLayerConfig | None = None
config_layer_config: DifyConfigLayerConfig | None = None
# Drive Skills & Files declaration (dify.drive) — an index the agent pulls
# through the back proxy, never inline content.
drive_config: DifyDriveLayerConfig | None = None
# Human-in-the-loop ask_human deferred tool (dify.ask_human). Present only when
# the Agent Soul configures human involvement (ENG-635).
ask_human_config: DifyAskHumanLayerConfig | None = None
@@ -305,7 +308,7 @@ class AgentBackendRunRequestBuilder:
"""Build an Agent App conversation-turn run request.
Layer graph: optional Agent Soul system prompt → user prompt →
execution context → optional shell / config / history
execution context → optional shell / config / drive / history
(multi-turn) → LLM → optional plugin-direct tools / core-routed tools /
knowledge search / ask_human / structured output. Mirrors the
workflow-node layer ordering minus the workflow-job / previous-node
@@ -330,9 +333,9 @@ class AgentBackendRunRequestBuilder:
[
RunLayerSpec(
name=AGENT_APP_USER_PROMPT_LAYER_ID,
type=DIFY_USER_PROMPT_LAYER_TYPE_ID,
type=PLAIN_PROMPT_LAYER_TYPE_ID,
metadata={**run_input.metadata, "origin": "agent_app_user_prompt"},
config=DifyUserPromptLayerConfig(text=run_input.user_prompt, files=run_input.user_files),
config=PromptLayerConfig(user=run_input.user_prompt),
),
RunLayerSpec(
name=DIFY_EXECUTION_CONTEXT_LAYER_ID,
@@ -343,7 +346,9 @@ class AgentBackendRunRequestBuilder:
]
)
include_shell = run_input.include_shell or run_input.config_layer_config is not None
include_shell = (
run_input.include_shell or run_input.config_layer_config is not None or run_input.drive_config is not None
)
if include_shell:
layers.append(
RunLayerSpec(
@@ -353,15 +358,16 @@ class AgentBackendRunRequestBuilder:
config=DifyRuntimeLayerConfig(backend_binding_ref=run_input.backend_binding_ref),
)
)
# Sandboxed bash workspace (dify.shell). It enters before config so
# eager pulls materialize content in the same filesystem used by model commands.
# Sandboxed bash workspace (dify.shell). It enters before config/drive
# so eager pulls materialize content in the same filesystem used by
# model commands.
layers.append(
RunLayerSpec(
name=DIFY_SHELL_LAYER_ID,
type=DIFY_SHELL_LAYER_TYPE_ID,
deps=_shell_layer_deps(),
metadata=run_input.metadata,
config=run_input.shell_config or DifyShellLayerConfig(),
config=_shell_config_with_drive_ref(run_input.shell_config, run_input.drive_config),
)
)
@@ -376,6 +382,19 @@ class AgentBackendRunRequestBuilder:
)
)
if run_input.drive_config is not None:
# Drive Skills & Files declaration (dify.drive): the catalog plus
# prompt-mentioned entries eagerly pulled through the shell layer.
layers.append(
RunLayerSpec(
name=DIFY_DRIVE_LAYER_ID,
type=DIFY_DRIVE_LAYER_TYPE_ID,
deps=_drive_layer_deps(),
metadata=run_input.metadata,
config=run_input.drive_config,
)
)
if run_input.include_history:
layers.append(
RunLayerSpec(
@@ -395,8 +414,8 @@ class AgentBackendRunRequestBuilder:
plugin_id=run_input.model.plugin_id,
model_provider=run_input.model.model_provider,
model=run_input.model.model,
credentials=run_input.model.credentials,
model_settings=_agent_model_settings(run_input.model.model_settings),
context_window_tokens=run_input.model.context_window_tokens,
),
)
)
@@ -426,8 +445,6 @@ class AgentBackendRunRequestBuilder:
)
)
if run_input.knowledge_fs is not None:
layers.append(_knowledge_fs_spec(run_input))
if run_input.knowledge is not None and run_input.knowledge.sets:
layers.append(
RunLayerSpec(
@@ -479,7 +496,7 @@ class AgentBackendRunRequestBuilder:
"""Build a workflow Agent Node run request without defining another wire schema.
Layer graph mirrors the workflow surface: prompts → execution context →
optional shell / config / history → LLM → optional
optional shell / config / drive / history → LLM → optional
plugin-direct tools / core-routed tools / knowledge search /
ask_human / structured output.
"""
@@ -521,7 +538,9 @@ class AgentBackendRunRequestBuilder:
]
)
include_shell = run_input.include_shell or run_input.config_layer_config is not None
include_shell = (
run_input.include_shell or run_input.config_layer_config is not None or run_input.drive_config is not None
)
if include_shell:
layers.append(
RunLayerSpec(
@@ -531,15 +550,16 @@ class AgentBackendRunRequestBuilder:
config=DifyRuntimeLayerConfig(backend_binding_ref=run_input.backend_binding_ref),
)
)
# Sandboxed bash workspace (dify.shell). It enters before config so
# eager pulls materialize content in the same filesystem used by model commands.
# Sandboxed bash workspace (dify.shell). It enters before drive so
# drive can materialize mentioned targets with `dify-agent drive pull`
# in the same shell-visible filesystem used by model commands.
layers.append(
RunLayerSpec(
name=DIFY_SHELL_LAYER_ID,
type=DIFY_SHELL_LAYER_TYPE_ID,
deps=_shell_layer_deps(),
metadata=run_input.metadata,
config=run_input.shell_config or DifyShellLayerConfig(),
config=_shell_config_with_drive_ref(run_input.shell_config, run_input.drive_config),
)
)
@@ -554,6 +574,19 @@ class AgentBackendRunRequestBuilder:
)
)
if run_input.drive_config is not None:
# Drive Skills & Files declaration (dify.drive): the catalog plus
# prompt-mentioned entries eagerly pulled through the shell layer.
layers.append(
RunLayerSpec(
name=DIFY_DRIVE_LAYER_ID,
type=DIFY_DRIVE_LAYER_TYPE_ID,
deps=_drive_layer_deps(),
metadata=run_input.metadata,
config=run_input.drive_config,
)
)
if run_input.include_history:
layers.append(
RunLayerSpec(
@@ -574,8 +607,8 @@ class AgentBackendRunRequestBuilder:
plugin_id=run_input.model.plugin_id,
model_provider=run_input.model.model_provider,
model=run_input.model.model,
credentials=run_input.model.credentials,
model_settings=_agent_model_settings(run_input.model.model_settings),
context_window_tokens=run_input.model.context_window_tokens,
),
),
]
@@ -606,8 +639,6 @@ class AgentBackendRunRequestBuilder:
)
)
if run_input.knowledge_fs is not None:
layers.append(_knowledge_fs_spec(run_input))
if run_input.knowledge is not None and run_input.knowledge.sets:
layers.append(
RunLayerSpec(
@@ -656,7 +687,7 @@ class AgentBackendRunRequestBuilder:
)
_SENSITIVE_KEY_PARTS = ("secret", "credential", "token", "password", "api_key", "base64_data")
_SENSITIVE_KEY_PARTS = ("secret", "credential", "token", "password", "api_key")
def redact_for_agent_backend_log(value: object) -> object:
@@ -665,10 +696,9 @@ def redact_for_agent_backend_log(value: object) -> object:
return redact_for_agent_backend_log(value.model_dump(mode="json", warnings=False))
if isinstance(value, dict):
redacted: dict[object, object] = {}
is_multimodal_file = value.get("type") == "image" and "filename" in value and "mime_type" in value
for key, item in value.items():
key_text = str(key).lower()
if any(part in key_text for part in _SENSITIVE_KEY_PARTS) or (is_multimodal_file and key_text == "url"):
if any(part in key_text for part in _SENSITIVE_KEY_PARTS):
redacted[key] = "[REDACTED]"
else:
redacted[key] = redact_for_agent_backend_log(item)

View File

@@ -10,7 +10,6 @@ from .data_migration import (
import_migration_data,
migration_data_wizard,
)
from .knowledge_fs import knowledge_fs_control_space
from .plugin import (
backfill_plugin_auto_upgrade,
extract_plugins,
@@ -23,13 +22,7 @@ from .plugin import (
setup_system_trigger_oauth_client,
transform_datasource_credentials,
)
from .rbac import (
migrate_agent_permissions_to_rbac,
migrate_dataset_permissions_to_rbac,
migrate_member_roles_to_rbac,
migrate_only_me_resource_whitelist_scopes_to_automatic_include,
migrate_resource_whitelist_scopes_to_automatic_include,
)
from .rbac import migrate_dataset_permissions_to_rbac, migrate_member_roles_to_rbac
from .retention import (
archive_workflow_runs,
archive_workflow_runs_plan,
@@ -82,17 +75,13 @@ __all__ = [
"import_migration_data",
"install_plugins",
"install_rag_pipeline_plugins",
"knowledge_fs_control_space",
"legacy_model_types",
"migrate_agent_permissions_to_rbac",
"migrate_annotation_vector_database",
"migrate_data_for_plugin",
"migrate_dataset_permissions_to_rbac",
"migrate_knowledge_vector_database",
"migrate_member_roles_to_rbac",
"migrate_only_me_resource_whitelist_scopes_to_automatic_include",
"migrate_oss",
"migrate_resource_whitelist_scopes_to_automatic_include",
"migration_data_wizard",
"old_metadata_migration",
"remove_orphaned_files_on_storage",

View File

@@ -8,7 +8,6 @@ from constants.languages import languages
from extensions.ext_database import db
from libs.helper import email as email_validate
from libs.password import hash_password, password_pattern, valid_password
from services.account_email import normalize_email
from services.account_service import AccountService, RegisterService, TenantService
@@ -21,7 +20,7 @@ def reset_password(email, new_password, password_confirm):
Reset password of owner account
Only available in SELF_HOSTED mode
"""
if new_password.strip() != password_confirm.strip():
if str(new_password).strip() != str(password_confirm).strip():
click.echo(click.style("Passwords do not match.", fg="red"))
return
normalized_email = email.strip().lower()
@@ -34,7 +33,7 @@ def reset_password(email, new_password, password_confirm):
try:
valid_password(new_password)
except ValueError:
except:
click.echo(click.style(f"Invalid password. Must match {password_pattern}", fg="red"))
return
@@ -63,7 +62,7 @@ def reset_email(email, new_email, email_confirm):
Replace account email
:return:
"""
if new_email.strip() != email_confirm.strip():
if str(new_email).strip() != str(email_confirm).strip():
click.echo(click.style("New emails do not match.", fg="red"))
return
normalized_new_email = new_email.strip().lower()
@@ -76,14 +75,13 @@ def reset_email(email, new_email, email_confirm):
try:
email_validate(normalized_new_email)
except ValueError:
except:
click.echo(click.style(f"Invalid email: {new_email}", fg="red"))
return
with Session(db.engine) as session:
account = session.merge(account)
account.email = normalized_new_email
account.normalized_email = normalize_email(normalized_new_email)
session.commit()
click.echo(click.style("Email updated successfully.", fg="green"))

View File

@@ -7,11 +7,7 @@ from typing import cast
import click
from commands.rbac import (
migrate_dataset_permissions_to_rbac,
migrate_only_me_resource_whitelist_scopes_to_automatic_include,
migrate_resource_whitelist_scopes_to_automatic_include, # noqa: F401
)
from commands.rbac import migrate_dataset_permissions_to_rbac
from extensions.ext_database import db
from graphon.model_runtime.entities.model_entities import ModelType
from services.legacy_model_type_migration import (
@@ -183,4 +179,3 @@ def legacy_model_types(
data_migrate.add_command(legacy_model_types)
data_migrate.add_command(migrate_dataset_permissions_to_rbac)
data_migrate.add_command(migrate_only_me_resource_whitelist_scopes_to_automatic_include)

View File

@@ -1,621 +0,0 @@
"""Operator commands for the independent KnowledgeFS control-plane."""
from __future__ import annotations
import json
from collections.abc import Callable
from datetime import datetime
from functools import partial
from pathlib import Path
import click
from pydantic import BaseModel, ValidationError
from core.db.session_factory import session_factory
from services.knowledge_fs.cleanup import (
CleanupApprovalInput,
CleanupCompletionEvidenceInput,
CleanupReadinessEvidenceInput,
CleanupStartInput,
KnowledgeFSCleanupError,
KnowledgeFSCleanupService,
)
from services.knowledge_fs.control_space_commands import KnowledgeFSControlSpaceCommandService
from services.knowledge_fs.control_space_lifecycle import KnowledgeFSControlSpaceLifecycleError
from services.knowledge_fs.control_space_management import (
KnowledgeFSControlSpaceManagementService,
KnowledgeFSControlSpaceRegistration,
)
from services.knowledge_fs.cutover import (
CutoverSmokeResultsInput,
FinalDeltaInput,
KnowledgeFSCutoverError,
KnowledgeFSWorkspaceCutoverService,
LegacyDependencyInput,
QuarantineResolutionInput,
ShadowAuthorizationObservationInput,
ShadowCompletionInput,
WorkspaceInventoryInput,
)
from services.knowledge_fs.greenfield_initializer import KnowledgeFSWorkspaceGreenfieldInitializer
from services.knowledge_fs.orphan_reconciler import KnowledgeFSOrphanReconciler
from services.knowledge_fs.remote_registry import get_knowledge_fs_lifecycle_remote
@click.group("knowledge-fs-control-space")
def knowledge_fs_control_space() -> None:
"""Inspect and repair Dify-owned KnowledgeFS control-space state."""
@knowledge_fs_control_space.command("dry-run")
@click.option("--tenant-id", default=None)
def dry_run(tenant_id: str | None) -> None:
report = _management_service().dry_run(tenant_id=tenant_id)
click.echo(json.dumps(report._asdict(), sort_keys=True))
@knowledge_fs_control_space.command("inventory")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Create ledgers; omitted means read-only inventory.")
def inventory(input_path: Path, apply: bool) -> None:
"""Validate strict Workspace inventory JSONL and optionally create ledgers."""
service = _cutover_service()
for payload in _read_jsonl(input_path, WorkspaceInventoryInput):
report = _operator_call(partial(service.inventory, payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("register")
@click.option("--tenant-id", required=True)
@click.option("--owner-account-id", required=True)
@click.option("--provisioning-key", required=True)
@click.option("--knowledge-space-id", required=True)
@click.option("--knowledge-space-revision", type=click.IntRange(min=0), required=True)
def register(
tenant_id: str,
owner_account_id: str,
provisioning_key: str,
knowledge_space_id: str,
knowledge_space_revision: int,
) -> None:
control_space, replayed = _management_service().register(
KnowledgeFSControlSpaceRegistration(
tenant_id,
owner_account_id,
provisioning_key,
knowledge_space_id,
knowledge_space_revision,
)
)
click.echo(json.dumps({"control_space_id": control_space.id, "replayed": replayed}, sort_keys=True))
@knowledge_fs_control_space.command("backfill")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist registrations; omitted means dry-run.")
def backfill(input_path: Path, apply: bool) -> None:
"""Backfill strict Workspace inventory JSONL; dry-run unless --apply is explicit."""
service = _cutover_service()
for payload in _read_jsonl(input_path, WorkspaceInventoryInput):
report = _operator_call(partial(service.backfill, payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("quarantine-resolve")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist resolutions; omitted means dry-run.")
def quarantine_resolve(input_path: Path, apply: bool) -> None:
"""Resolve strict tenant-scoped quarantine JSONL with immutable operator evidence."""
service = _cutover_service()
for payload in _read_jsonl(input_path, QuarantineResolutionInput):
report = _operator_call(partial(service.resolve_quarantine, payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("shadow-start")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--at", "started_at", default=None, help="Optional explicit timezone-aware shadow start.")
def shadow_start(tenant_id: str, expected_cas_version: int, started_at: str | None) -> None:
service = _cutover_service()
_operator_call(
lambda: service.begin_shadow(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
started_at=_parse_timestamp(started_at) if started_at is not None else None,
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("shadow-report")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist observations; omitted means dry-run.")
def shadow_report(input_path: Path, apply: bool) -> None:
observations = _read_jsonl(input_path, ShadowAuthorizationObservationInput)
report = _operator_call(lambda: _cutover_service().record_shadow_report(observations, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("shadow-complete")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist completion; omitted means dry-run.")
def shadow_complete(input_path: Path, apply: bool) -> None:
payload = _read_one_jsonl(input_path, ShadowCompletionInput)
report = _operator_call(lambda: _cutover_service().complete_shadow(payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("issue-approve")
@click.option("--tenant-id", required=True)
@click.option("--issue-key", required=True)
@click.option("--account-id", required=True)
@click.option("--at", "approved_at", required=True)
def issue_approve(tenant_id: str, issue_key: str, account_id: str, approved_at: str) -> None:
_operator_call(
lambda: _cutover_service().approve_issue_fail_closed(
tenant_id=tenant_id,
issue_key=issue_key,
account_id=account_id,
approved_at=_parse_timestamp(approved_at),
)
)
_echo_json(_cutover_service().status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("issue-resolve")
@click.option("--tenant-id", required=True)
@click.option("--issue-key", required=True)
@click.option("--account-id", required=True)
@click.option("--at", "resolved_at", required=True)
def issue_resolve(tenant_id: str, issue_key: str, account_id: str, resolved_at: str) -> None:
_operator_call(
lambda: _cutover_service().resolve_issue(
tenant_id=tenant_id,
issue_key=issue_key,
account_id=account_id,
resolved_at=_parse_timestamp(resolved_at),
)
)
_echo_json(_cutover_service().status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("shadow-approve")
@click.option("--tenant-id", required=True)
@click.option("--diff-key", required=True)
@click.option("--account-id", required=True)
@click.option("--at", "approved_at", required=True)
def shadow_approve(tenant_id: str, diff_key: str, account_id: str, approved_at: str) -> None:
_operator_call(
lambda: _cutover_service().approve_shadow_diff(
tenant_id=tenant_id,
diff_key=diff_key,
account_id=account_id,
approved_at=_parse_timestamp(approved_at),
)
)
_echo_json(_cutover_service().status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("shadow-resolve")
@click.option("--tenant-id", required=True)
@click.option("--diff-key", required=True)
@click.option("--account-id", required=True)
@click.option("--at", "resolved_at", required=True)
def shadow_resolve(tenant_id: str, diff_key: str, account_id: str, resolved_at: str) -> None:
_operator_call(
lambda: _cutover_service().resolve_shadow_diff(
tenant_id=tenant_id,
diff_key=diff_key,
account_id=account_id,
resolved_at=_parse_timestamp(resolved_at),
)
)
_echo_json(_cutover_service().status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("legacy-dashboard")
@click.option("--tenant-id", required=True)
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--checked-at", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), default=None)
@click.option("--apply", is_flag=True, default=False, help="Persist gate evidence; omitted means read-only report.")
def legacy_dashboard(
tenant_id: str,
input_path: Path,
checked_at: str,
expected_cas_version: int | None,
apply: bool,
) -> None:
dependencies = _read_jsonl(input_path, LegacyDependencyInput, allow_empty=True)
report = _operator_call(
lambda: _cutover_service().legacy_dependency_dashboard(
tenant_id=tenant_id,
dependencies=dependencies,
expected_cas_version=expected_cas_version,
checked_at=_parse_timestamp(checked_at),
apply=apply,
)
)
_echo_json(report._asdict())
@knowledge_fs_control_space.command("legacy-check")
@click.option("--tenant-id", required=True)
def legacy_check(tenant_id: str) -> None:
status_report = _operator_call(lambda: _cutover_service().status(tenant_id=tenant_id))
passed = (
bool(status_report["legacy_dependency_ready"])
and status_report["open_issues"] == 0
and status_report["unresolved_cutover_quarantine"] == 0
)
_echo_json({"tenant_id": tenant_id, "passed": passed, "status": status_report})
if not passed:
raise click.exceptions.Exit(1)
@knowledge_fs_control_space.command("freeze")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--at", "freeze_at", required=True)
def freeze(tenant_id: str, expected_cas_version: int, freeze_at: str) -> None:
service = _cutover_service()
_operator_call(
lambda: service.freeze(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
freeze_at=_parse_timestamp(freeze_at),
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("final-delta")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
def final_delta(input_path: Path) -> None:
payload = _read_one_jsonl(input_path, FinalDeltaInput)
service = _cutover_service()
_operator_call(lambda: service.apply_final_delta(payload))
_echo_json(service.status(tenant_id=str(payload.tenant_id)))
@knowledge_fs_control_space.command("cutover")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--at", "cutover_at", required=True)
@click.option("--rollback-cutoff-at", required=True)
def cutover(tenant_id: str, expected_cas_version: int, cutover_at: str, rollback_cutoff_at: str) -> None:
service = _cutover_service()
_operator_call(
lambda: service.cutover(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
cutover_at=_parse_timestamp(cutover_at),
rollback_cutoff_at=_parse_timestamp(rollback_cutoff_at),
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("smoke")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
def smoke(tenant_id: str, expected_cas_version: int, input_path: Path) -> None:
results = _read_one_jsonl(input_path, CutoverSmokeResultsInput)
service = _cutover_service()
_operator_call(
lambda: service.record_smoke_results(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
results=results,
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("observe")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--started-at", default=None)
@click.option("--window-ends-at", default=None)
@click.option("--maximum-task-expires-at", default=None)
@click.option("--observed-at", default=None)
def observe(
tenant_id: str,
expected_cas_version: int,
started_at: str | None,
window_ends_at: str | None,
maximum_task_expires_at: str | None,
observed_at: str | None,
) -> None:
service = _cutover_service()
if observed_at is not None:
if any(value is not None for value in (started_at, window_ends_at, maximum_task_expires_at)):
raise click.UsageError("--observed-at cannot be combined with observation start options")
_operator_call(
lambda: service.complete_observation(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
observed_at=_parse_timestamp(observed_at),
)
)
else:
if started_at is None or window_ends_at is None or maximum_task_expires_at is None:
raise click.UsageError(
"observation start requires --started-at, --window-ends-at, and --maximum-task-expires-at"
)
_operator_call(
lambda: service.begin_observation(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
started_at=_parse_timestamp(started_at),
window_ends_at=_parse_timestamp(window_ends_at),
maximum_task_expires_at=_parse_timestamp(maximum_task_expires_at),
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("rollback")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--at", "rolled_back_at", required=True)
def rollback(tenant_id: str, expected_cas_version: int, rolled_back_at: str) -> None:
service = _cutover_service()
_operator_call(
lambda: service.rollback(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
rolled_back_at=_parse_timestamp(rolled_back_at),
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("status")
@click.option("--tenant-id", required=True)
def status(tenant_id: str) -> None:
_echo_json(_operator_call(lambda: _cutover_service().status(tenant_id=tenant_id)))
@knowledge_fs_control_space.command("greenfield-initialize")
@click.option("--tenant-id", required=True)
def greenfield_initialize(tenant_id: str) -> None:
"""Idempotently initialize one Workspace that has no KnowledgeFS state."""
_operator_call(lambda: _greenfield_initializer().ensure_initialized(tenant_id=tenant_id))
_echo_json(_operator_call(lambda: _cutover_service().status(tenant_id=tenant_id)))
@knowledge_fs_control_space.command("cleanup-request")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist readiness evidence; omitted means dry-run.")
def cleanup_request(input_path: Path, apply: bool) -> None:
payload = _read_one_jsonl(input_path, CleanupReadinessEvidenceInput)
report = _cleanup_call(lambda: _cleanup_service().request_cleanup(payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("cleanup-approve")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist four-eyes approval; omitted means dry-run.")
def cleanup_approve(input_path: Path, apply: bool) -> None:
payload = _read_one_jsonl(input_path, CleanupApprovalInput)
report = _cleanup_call(lambda: _cleanup_service().approve_cleanup(payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("cleanup-start")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist the irreversible fence; never runs deletion.")
@click.option(
"--acknowledge-irreversible",
is_flag=True,
default=False,
help="Required with --apply; confirms rollback will be permanently closed.",
)
def cleanup_start(input_path: Path, apply: bool, acknowledge_irreversible: bool) -> None:
payload = _read_one_jsonl(input_path, CleanupStartInput)
if apply and not acknowledge_irreversible:
raise click.UsageError("--apply requires --acknowledge-irreversible")
report = _cleanup_call(lambda: _cleanup_service().start_cleanup(payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("cleanup-complete")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist externally verified cleanup completion.")
@click.option(
"--acknowledge-executed",
is_flag=True,
default=False,
help="Required with --apply; confirms the reviewed destructive bundle already executed.",
)
def cleanup_complete(input_path: Path, apply: bool, acknowledge_executed: bool) -> None:
payload = _read_one_jsonl(input_path, CleanupCompletionEvidenceInput)
if apply and not acknowledge_executed:
raise click.UsageError("--apply requires --acknowledge-executed")
report = _cleanup_call(lambda: _cleanup_service().complete_cleanup(payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("cleanup-status")
@click.option("--tenant-id", required=True)
@click.option("--request-id", required=True)
def cleanup_status(tenant_id: str, request_id: str) -> None:
_echo_json(_cleanup_call(lambda: _cleanup_service().status(tenant_id=tenant_id, request_id=request_id)))
@knowledge_fs_control_space.command("repair")
@click.option("--tenant-id", required=True)
@click.option("--control-space-id", required=True)
@click.option("--expected-resource-version", type=click.IntRange(min=0), required=True)
@click.option("--knowledge-space-id", required=True)
@click.option("--knowledge-space-revision", type=click.IntRange(min=0), required=True)
def repair(
tenant_id: str,
control_space_id: str,
expected_resource_version: int,
knowledge_space_id: str,
knowledge_space_revision: int,
) -> None:
control_space = _management_service().repair_registration(
tenant_id=tenant_id,
control_space_id=control_space_id,
expected_resource_version=expected_resource_version,
knowledge_space_id=knowledge_space_id,
knowledge_space_revision=knowledge_space_revision,
)
click.echo(json.dumps({"control_space_id": control_space.id, "state": control_space.state.value}, sort_keys=True))
@knowledge_fs_control_space.command("orphan-report")
@click.option("--limit", type=click.IntRange(min=1, max=10_000), default=500, show_default=True)
def orphan_report(limit: int) -> None:
report = KnowledgeFSOrphanReconciler(
session_factory.get_session_maker(),
get_knowledge_fs_lifecycle_remote(),
).reconcile(limit=limit, apply_repairs=False)
click.echo(json.dumps(report._asdict(), sort_keys=True))
@knowledge_fs_control_space.command("workspace-delete-request")
@click.option("--tenant-id", required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist durable deletion intents; omitted means dry-run.")
def workspace_delete_request(tenant_id: str, apply: bool) -> None:
"""Route every KnowledgeFS Space through the canonical lifecycle deletion path."""
if not apply:
report = _management_service().dry_run(tenant_id=tenant_id)
_echo_json(
{
"apply": False,
"by_state": report.by_state,
"tenant_id": tenant_id,
"total": report.total,
}
)
return
results = _lifecycle_call(lambda: _lifecycle_service().request_workspace_cleanup(tenant_id=tenant_id))
_echo_json(
{
"apply": True,
"control_space_ids": [result.control_space.id for result in results],
"operation_ids": [result.outbox.operation_id for result in results if result.outbox is not None],
"tenant_id": tenant_id,
}
)
@knowledge_fs_control_space.command("workspace-delete-finalize")
@click.option("--tenant-id", required=True)
@click.option("--apply", is_flag=True, default=False, help="Purge terminal local control-plane rows.")
@click.option(
"--acknowledge-control-plane-purge",
is_flag=True,
default=False,
help="Required with --apply after every remote Space has reached deleted.",
)
def workspace_delete_finalize(tenant_id: str, apply: bool, acknowledge_control_plane_purge: bool) -> None:
"""Release the Workspace FK only after all remote deletions are terminal."""
service = _lifecycle_service()
if not apply:
_lifecycle_call(lambda: service.assert_workspace_deletion_allowed(tenant_id=tenant_id))
_echo_json({"apply": False, "ready": True, "tenant_id": tenant_id})
return
if not acknowledge_control_plane_purge:
raise click.UsageError("--apply requires --acknowledge-control-plane-purge")
deleted = _lifecycle_call(lambda: service.finalize_workspace_deletion(tenant_id=tenant_id))
_echo_json({"apply": True, "purged_control_spaces": deleted, "tenant_id": tenant_id})
def _read_jsonl[InputT: BaseModel](
input_path: Path, input_type: type[InputT], *, allow_empty: bool = False
) -> tuple[InputT, ...]:
records: list[InputT] = []
for line_number, line in enumerate(input_path.read_text(encoding="utf-8").splitlines(), start=1):
if not line.strip():
continue
try:
records.append(input_type.model_validate_json(line))
except ValidationError as exc:
raise click.ClickException(f"invalid strict JSONL at line {line_number}: {exc}") from exc
if not records and not allow_empty:
raise click.ClickException("strict JSONL input must contain at least one record")
return tuple(records)
def _read_one_jsonl[InputT: BaseModel](input_path: Path, input_type: type[InputT]) -> InputT:
records = _read_jsonl(input_path, input_type)
if len(records) != 1:
raise click.ClickException("this command requires exactly one JSONL record")
return records[0]
def _parse_timestamp(value: str) -> datetime:
try:
parsed = datetime.fromisoformat(value)
except ValueError as exc:
raise click.ClickException(f"invalid ISO-8601 timestamp: {value}") from exc
if parsed.tzinfo is None:
raise click.ClickException("operator timestamps must include an explicit timezone")
return parsed
def _operator_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
try:
return operation()
except KnowledgeFSCutoverError as exc:
raise click.ClickException(str(exc)) from exc
def _cleanup_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
try:
return operation()
except KnowledgeFSCleanupError as exc:
raise click.ClickException(str(exc)) from exc
def _lifecycle_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
try:
return operation()
except KnowledgeFSControlSpaceLifecycleError as exc:
raise click.ClickException(str(exc)) from exc
def _echo_json(payload: object) -> None:
click.echo(json.dumps(payload, default=str, sort_keys=True))
def _management_service() -> KnowledgeFSControlSpaceManagementService:
return KnowledgeFSControlSpaceManagementService(session_factory.get_session_maker())
def _cutover_service() -> KnowledgeFSWorkspaceCutoverService:
return KnowledgeFSWorkspaceCutoverService(
session_factory.get_session_maker(),
remote_factory=get_knowledge_fs_lifecycle_remote,
)
def _greenfield_initializer() -> KnowledgeFSWorkspaceGreenfieldInitializer:
return KnowledgeFSWorkspaceGreenfieldInitializer(
session_factory.get_session_maker(),
cutover=_cutover_service(),
)
def _cleanup_service() -> KnowledgeFSCleanupService:
return KnowledgeFSCleanupService(session_factory.get_session_maker())
def _lifecycle_service() -> KnowledgeFSControlSpaceCommandService:
return KnowledgeFSControlSpaceCommandService(session_factory.get_session_maker())
__all__ = ["knowledge_fs_control_space"]

View File

@@ -3,8 +3,6 @@ from __future__ import annotations
import json
from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
from enum import StrEnum
import click
from sqlalchemy import select
@@ -13,33 +11,10 @@ from sqlalchemy.orm import Session
from configs import dify_config
from core.db.session_factory import session_factory
from core.rbac import RBACResourceWhitelistScope
from models import (
Agent,
AgentStatus,
App,
Dataset,
DatasetPermission,
DatasetPermissionEnum,
Tenant,
TenantAccountJoin,
TenantAccountRole,
)
from services.enterprise.rbac_service import (
LegacyAgentRoleMigration,
ListOption,
RBACResourceType,
RBACService,
ReplaceMemberBindings,
ReplaceUserAccessPolicies,
)
from models import Dataset, DatasetPermission, DatasetPermissionEnum, TenantAccountJoin, TenantAccountRole
from services.enterprise.rbac_service import ListOption, RBACService, ReplaceMemberBindings, ReplaceUserAccessPolicies
_RBAC_DEFAULT_ACCESS_POLICY_ID = "default"
_RBAC_RESOURCE_ACCESS_POLICY_BATCH_SIZE = 500
_AGENT_MIGRATION_STATE_LABEL = {
False: "would change",
True: "changed",
}
_LEGACY_ROLE_TO_BUILTIN_TAG = {
TenantAccountRole.OWNER.value: "owner",
@@ -91,88 +66,6 @@ def _resolve_builtin_role_id(tenant_id: str, operator_account_id: str, legacy_ro
return _resolve_builtin_role_ids(tenant_id, operator_account_id)[legacy_role]
def _iter_tenant_ids(tenant_id: str | None, *, batch_size: int) -> Iterator[str]:
if tenant_id:
yield tenant_id
return
last_id: str | None = None
while True:
with session_factory.create_session() as session:
stmt = select(Tenant.id).order_by(Tenant.id.asc()).limit(batch_size)
if last_id is not None:
stmt = stmt.where(Tenant.id > last_id)
rows = session.execute(stmt).scalars().all()
if not rows:
return
for row in rows:
yield str(row)
last_id = str(rows[-1])
def _emit_agent_migration_event(payload: dict[str, object]) -> None:
click.echo(json.dumps(payload, sort_keys=True))
@dataclass(frozen=True)
class _AgentMigrationEventKind:
event_stem: str
include_tenant_id: bool = True
@property
def skipped(self) -> str:
return f"{self.event_stem}_skipped"
@property
def failed(self) -> str:
return f"{self.event_stem}_failed"
def outcome(self, *, apply: bool) -> str:
return f"{self.event_stem}_applied" if apply else f"{self.event_stem}_proposed_change"
_AGENT_ROLE_MIGRATION_EVENT_KIND = _AgentMigrationEventKind(event_stem="agent_manage_role_migration")
_AGENT_ROLE_TEMPLATE_MIGRATION_EVENT_KIND = _AgentMigrationEventKind(
event_stem="agent_manage_role_template_migration", include_tenant_id=False
)
_AGENT_ACCESS_BOOTSTRAP_EVENT_KIND = _AgentMigrationEventKind(event_stem="agent_access_bootstrap")
_AGENT_BACKING_APP_SPECIFIC_WHITELIST_EVENT = "agent_backing_app_has_specific_whitelist"
_AGENT_ACCESS_BOOTSTRAP_MEMBER_SOURCE = "workspace_members"
class _AgentAccessBootstrapReason(StrEnum):
ALREADY_INITIALIZED = "already_initialized"
MISSING_WHITELIST_SCOPE = "missing_whitelist_scope"
NO_CREATOR = "no_creator"
def _agent_manage_role_event(
tenant_id: str | None,
entry: LegacyAgentRoleMigration,
*,
apply: bool,
kind: _AgentMigrationEventKind,
) -> dict[str, object]:
base: dict[str, object] = {
"dry_run": not apply,
"role_id": entry.role_id,
"role_name": entry.role_name,
}
if kind.include_tenant_id:
base["tenant_id"] = tenant_id
if entry.skipped:
return {**base, "event": kind.skipped, "reason": entry.skipped}
return {
**base,
"event": kind.outcome(apply=apply),
"after": {
"added_keys": entry.added_keys,
"removed_keys": entry.removed_keys,
"bound_policies": entry.bound_policies,
},
}
def _iter_tenant_member_batches(
tenant_id: str | None,
*,
@@ -417,499 +310,6 @@ def _emit_dataset_permission_migration_event(payload: dict[str, object]) -> None
click.echo(json.dumps(payload, sort_keys=True))
def _emit_resource_whitelist_scope_migration_event(payload: dict[str, object]) -> None:
click.echo(json.dumps(payload, sort_keys=True))
def _normalize_rbac_whitelist_scope(scope: object) -> RBACResourceWhitelistScope | None:
if scope is None:
return None
try:
return RBACResourceWhitelistScope(str(scope))
except ValueError as exc:
raise ValueError(f"Unsupported RBAC whitelist scope: {scope}") from exc
def _owner_account_id(tenant_id: str, *, session: Session) -> str:
account_id = session.scalar(
select(TenantAccountJoin.account_id)
.where(TenantAccountJoin.tenant_id == tenant_id, TenantAccountJoin.role == TenantAccountRole.OWNER)
.order_by(TenantAccountJoin.id.asc())
.limit(1)
)
if not account_id:
raise ValueError(f"Workspace owner not found for tenant={tenant_id}")
return str(account_id)
def _workspace_member_account_id_batches(tenant_id: str, batch_size: int) -> Iterator[list[str]]:
last_join_id: str | None = None
while True:
with session_factory.create_session() as session:
stmt = (
select(TenantAccountJoin.id, TenantAccountJoin.account_id)
.where(TenantAccountJoin.tenant_id == tenant_id)
.order_by(TenantAccountJoin.id.asc())
.limit(batch_size)
)
if last_join_id:
stmt = stmt.where(TenantAccountJoin.id > last_join_id)
rows = list(session.execute(stmt).all())
if not rows:
return
yield [str(row.account_id) for row in rows]
last_join_id = str(rows[-1].id)
def _replace_resource_whitelist(
resource_type: str,
*,
tenant_id: str,
operator_account_id: str,
resource_id: str,
automatic_include_workspace_members: bool,
) -> None:
payload = ReplaceMemberBindings(automatic_include_workspace_members=automatic_include_workspace_members)
if resource_type == "app":
RBACService.AppAccess.replace_whitelist(
tenant_id=tenant_id,
account_id=operator_account_id,
app_id=resource_id,
payload=payload,
)
return
RBACService.DatasetAccess.replace_whitelist(
tenant_id=tenant_id,
account_id=operator_account_id,
dataset_id=resource_id,
payload=payload,
)
def _replace_resource_default_access_policies(
resource_type: str,
*,
tenant_id: str,
operator_account_id: str,
resource_id: str,
account_ids: list[str],
) -> None:
if not account_ids:
return
payload = ReplaceUserAccessPolicies(
access_policy_ids=[_RBAC_DEFAULT_ACCESS_POLICY_ID],
account_ids=account_ids,
)
if resource_type == "app":
RBACService.AppAccess.replace_user_access_policies(
tenant_id=tenant_id,
account_id=operator_account_id,
app_id=resource_id,
target_account_id=None,
payload=payload,
)
return
RBACService.DatasetAccess.replace_user_access_policies(
tenant_id=tenant_id,
account_id=operator_account_id,
dataset_id=resource_id,
target_account_id=None,
payload=payload,
)
def _resource_legacy_whitelist_config(
resource_type: str,
*,
tenant_id: str,
operator_account_id: str,
resource_id: str,
):
if resource_type == "app":
return RBACService.AppAccess.legacy_whitelist_config(
tenant_id=tenant_id,
account_id=operator_account_id,
app_id=resource_id,
)
return RBACService.DatasetAccess.legacy_whitelist_config(
tenant_id=tenant_id,
account_id=operator_account_id,
dataset_id=resource_id,
)
def _iter_rbac_resource_rows(
resource_type: str,
*,
tenant_id: str | None,
resource_id: str | None,
batch_size: int,
) -> Iterator[tuple[str, str, str, str | None]]:
model = App if resource_type == "app" else Dataset
last_resource_id: str | None = None
while True:
with session_factory.create_session() as session:
stmt = select(model.id, model.tenant_id, model.maintainer).order_by(model.id.asc()).limit(batch_size)
if tenant_id:
stmt = stmt.where(model.tenant_id == tenant_id)
if resource_id:
stmt = stmt.where(model.id == resource_id)
if last_resource_id:
stmt = stmt.where(model.id > last_resource_id)
rows = list(session.execute(stmt).all())
if not rows:
return
for row in rows:
yield resource_type, str(row.tenant_id), str(row.id), str(row.maintainer) if row.maintainer else None
last_resource_id = str(rows[-1].id)
if resource_id:
return
def _iter_selected_rbac_resource_rows(
resource_type: str,
*,
tenant_id: str | None,
resource_id: str | None,
batch_size: int,
) -> Iterator[tuple[str, str, str, str | None]]:
if resource_type in {"app", "all"}:
yield from _iter_rbac_resource_rows(
"app",
tenant_id=tenant_id,
resource_id=resource_id if resource_type == "app" else None,
batch_size=batch_size,
)
if resource_type in {"dataset", "all"}:
yield from _iter_rbac_resource_rows(
"dataset",
tenant_id=tenant_id,
resource_id=resource_id if resource_type == "dataset" else None,
batch_size=batch_size,
)
@click.command(
"rbac-migrate-resource-whitelist-scopes",
help=(
"Migrate RBAC app/dataset whitelist scope values to automatic_include_workspace_members. "
"Old scope all becomes true; specific and only_me become false."
),
)
@click.option("--tenant-id", help="Only migrate resources in a single workspace.")
@click.option(
"--resource-type",
type=click.Choice(["app", "dataset", "all"]),
default="all",
show_default=True,
help="Resource type to migrate.",
)
@click.option("--resource-id", help="Only migrate a single resource. Requires --resource-type app or dataset.")
@click.option("--batch-size", default=500, show_default=True, type=click.IntRange(min=1))
@click.option(
"--member-batch-size",
default=_RBAC_RESOURCE_ACCESS_POLICY_BATCH_SIZE,
show_default=True,
type=click.IntRange(min=1),
help="Workspace members written per default-policy call when migrating old scope all.",
)
@click.option(
"--dry-run/--apply",
default=True,
show_default=True,
help="Preview the migration without writing RBAC bindings. Use --apply to write changes.",
)
def migrate_resource_whitelist_scopes_to_automatic_include(
tenant_id: str | None,
resource_type: str,
resource_id: str | None,
batch_size: int,
member_batch_size: int,
dry_run: bool,
) -> None:
"""Backfill RBAC app/dataset automatic workspace-member inclusion from old RBAC scope."""
if resource_id and resource_type == "all":
raise click.BadParameter("--resource-id requires --resource-type app or dataset", param_hint="--resource-id")
click.echo(click.style("Starting RBAC resource whitelist scope migration.", fg="green"))
scanned_count = 0
migrated_count = 0
member_policy_batch_count = 0
owner_account_ids_by_tenant_id: dict[str, str] = {}
for (
current_resource_type,
workspace_id,
current_resource_id,
maintainer_account_id,
) in _iter_selected_rbac_resource_rows(
resource_type,
tenant_id=tenant_id,
resource_id=resource_id,
batch_size=batch_size,
):
scanned_count += 1
with session_factory.create_session() as session:
operator_account_id = maintainer_account_id or owner_account_ids_by_tenant_id.get(workspace_id)
if not operator_account_id:
operator_account_id = _owner_account_id(workspace_id, session=session)
owner_account_ids_by_tenant_id[workspace_id] = operator_account_id
legacy_config = _resource_legacy_whitelist_config(
current_resource_type,
tenant_id=workspace_id,
operator_account_id=operator_account_id,
resource_id=current_resource_id,
)
scope = _normalize_rbac_whitelist_scope(legacy_config.rbac_whitelist_scope)
if scope is None:
_emit_resource_whitelist_scope_migration_event(
{
"event": "resource_whitelist_scope_migration_skipped",
"reason": "missing_legacy_scope",
"dry_run": dry_run,
"tenant_id": workspace_id,
"resource_type": current_resource_type,
"resource_id": current_resource_id,
}
)
continue
automatic_include_workspace_members = scope is RBACResourceWhitelistScope.ALL
if scope is RBACResourceWhitelistScope.SPECIFIC:
member_account_ids = sorted(set(legacy_config.account_ids))
member_source = "legacy_specific_members"
elif scope is RBACResourceWhitelistScope.ONLY_ME:
member_account_ids = [maintainer_account_id] if maintainer_account_id else []
member_source = "resource_maintainer"
else:
member_account_ids = []
member_source = "workspace_members"
_emit_resource_whitelist_scope_migration_event(
{
"event": "resource_whitelist_scope_migration_proposed_change",
"dry_run": dry_run,
"tenant_id": workspace_id,
"operator_account_id": operator_account_id,
"resource_type": current_resource_type,
"resource_id": current_resource_id,
"before": {
"rbac_whitelist_scope": scope.value,
"legacy_account_ids": sorted(set(legacy_config.account_ids)),
},
"after": {
"automatic_include_workspace_members": automatic_include_workspace_members,
"default_policy_member_source": member_source,
"default_policy_account_ids": member_account_ids,
},
}
)
if dry_run:
continue
_replace_resource_whitelist(
current_resource_type,
tenant_id=workspace_id,
operator_account_id=operator_account_id,
resource_id=current_resource_id,
automatic_include_workspace_members=automatic_include_workspace_members,
)
migrated_count += 1
if scope is RBACResourceWhitelistScope.ALL:
for batch in _workspace_member_account_id_batches(workspace_id, member_batch_size):
_replace_resource_default_access_policies(
current_resource_type,
tenant_id=workspace_id,
operator_account_id=operator_account_id,
resource_id=current_resource_id,
account_ids=batch,
)
member_policy_batch_count += 1
else:
_replace_resource_default_access_policies(
current_resource_type,
tenant_id=workspace_id,
operator_account_id=operator_account_id,
resource_id=current_resource_id,
account_ids=member_account_ids,
)
if member_account_ids:
member_policy_batch_count += 1
if scanned_count == 0:
click.echo(click.style("No RBAC resources found for migration.", fg="yellow"))
return
if dry_run:
click.echo(
click.style(
f"Dry run completed. Scanned {scanned_count} RBAC resources. No RBAC bindings were written.",
fg="yellow",
)
)
else:
click.echo(
click.style(
"RBAC resource whitelist scope migration completed. "
f"Scanned {scanned_count} resources, migrated {migrated_count}, "
f"wrote {member_policy_batch_count} default-policy batches.",
fg="green",
)
)
@click.command(
"rbac-migrate-resource-whitelist-scopes",
help=(
"Migrate RBAC app/dataset whitelist configs whose old scope is only_me to "
"automatic_include_workspace_members=true and sync workspace members into the whitelist."
),
)
@click.option("--tenant-id", help="Only migrate resources in a single workspace.")
@click.option(
"--resource-type",
type=click.Choice(["app", "dataset", "all"]),
default="all",
show_default=True,
help="Resource type to migrate.",
)
@click.option("--resource-id", help="Only migrate a single resource. Requires --resource-type app or dataset.")
@click.option("--batch-size", default=500, show_default=True, type=click.IntRange(min=1))
@click.option(
"--member-batch-size",
default=_RBAC_RESOURCE_ACCESS_POLICY_BATCH_SIZE,
show_default=True,
type=click.IntRange(min=1),
help="Workspace members written per default-policy call.",
)
@click.option(
"--dry-run/--apply",
default=True,
show_default=True,
help="Preview the migration without writing RBAC bindings. Use --apply to write changes.",
)
def migrate_only_me_resource_whitelist_scopes_to_automatic_include(
tenant_id: str | None,
resource_type: str,
resource_id: str | None,
batch_size: int,
member_batch_size: int,
dry_run: bool,
) -> None:
"""Backfill automatic workspace-member inclusion for old RBAC only_me resource scopes."""
if resource_id and resource_type == "all":
raise click.BadParameter("--resource-id requires --resource-type app or dataset", param_hint="--resource-id")
click.echo(click.style("Starting RBAC only_me resource whitelist scope migration.", fg="green"))
scanned_count = 0
only_me_count = 0
migrated_count = 0
member_policy_batch_count = 0
owner_account_ids_by_tenant_id: dict[str, str] = {}
for (
current_resource_type,
workspace_id,
current_resource_id,
maintainer_account_id,
) in _iter_selected_rbac_resource_rows(
resource_type,
tenant_id=tenant_id,
resource_id=resource_id,
batch_size=batch_size,
):
scanned_count += 1
with session_factory.create_session() as session:
operator_account_id = maintainer_account_id or owner_account_ids_by_tenant_id.get(workspace_id)
if not operator_account_id:
operator_account_id = _owner_account_id(workspace_id, session=session)
owner_account_ids_by_tenant_id[workspace_id] = operator_account_id
legacy_config = _resource_legacy_whitelist_config(
current_resource_type,
tenant_id=workspace_id,
operator_account_id=operator_account_id,
resource_id=current_resource_id,
)
scope = _normalize_rbac_whitelist_scope(legacy_config.rbac_whitelist_scope)
if scope is not RBACResourceWhitelistScope.ONLY_ME:
continue
only_me_count += 1
_emit_resource_whitelist_scope_migration_event(
{
"event": "only_me_resource_whitelist_scope_migration_proposed_change",
"dry_run": dry_run,
"tenant_id": workspace_id,
"operator_account_id": operator_account_id,
"resource_type": current_resource_type,
"resource_id": current_resource_id,
"before": {
"rbac_whitelist_scope": scope.value,
"legacy_account_ids": sorted(set(legacy_config.account_ids)),
},
"after": {
"automatic_include_workspace_members": True,
"default_policy_member_source": "workspace_members",
},
}
)
if dry_run:
continue
_replace_resource_whitelist(
current_resource_type,
tenant_id=workspace_id,
operator_account_id=operator_account_id,
resource_id=current_resource_id,
automatic_include_workspace_members=True,
)
migrated_count += 1
for batch in _workspace_member_account_id_batches(workspace_id, member_batch_size):
_replace_resource_default_access_policies(
current_resource_type,
tenant_id=workspace_id,
operator_account_id=operator_account_id,
resource_id=current_resource_id,
account_ids=batch,
)
member_policy_batch_count += 1
if scanned_count == 0:
click.echo(click.style("No RBAC resources found for migration.", fg="yellow"))
return
if dry_run:
click.echo(
click.style(
f"Dry run completed. Scanned {scanned_count} RBAC resources, found {only_me_count} only_me resources. "
"No RBAC bindings were written.",
fg="yellow",
)
)
else:
click.echo(
click.style(
"RBAC only_me resource whitelist scope migration completed. "
f"Scanned {scanned_count} resources, migrated {migrated_count}, "
f"wrote {member_policy_batch_count} default-policy batches.",
fg="green",
)
)
@click.command(
"rbac-migrate-dataset-permissions",
help=(
@@ -999,9 +399,7 @@ def migrate_dataset_permissions_to_rbac(
)
scanned_count += 1
replace_whitelist_payload = ReplaceMemberBindings(
automatic_include_workspace_members=scope is RBACResourceWhitelistScope.ALL
)
replace_whitelist_payload = ReplaceMemberBindings(scope=scope)
if dry_run:
_emit_dataset_permission_migration_event(
{
@@ -1111,302 +509,3 @@ def migrate_dataset_permissions_to_rbac(
fg="green",
)
)
_AgentRow = tuple[str, str | None, str | None]
def _iter_agent_row_batches(tenant_id: str, batch_size: int) -> Iterator[list[_AgentRow]]:
last_agent_id: str | None = None
while True:
with session_factory.create_session() as session:
stmt = (
select(Agent.id, Agent.created_by, Agent.backing_app_id)
.where(Agent.tenant_id == tenant_id, Agent.status != AgentStatus.ARCHIVED)
.order_by(Agent.id.asc())
.limit(batch_size)
)
if last_agent_id:
stmt = stmt.where(Agent.id > last_agent_id)
rows = list(session.execute(stmt).all())
if not rows:
return
yield [
(
str(row.id),
str(row.created_by) if row.created_by else None,
str(row.backing_app_id) if row.backing_app_id else None,
)
for row in rows
]
last_agent_id = str(rows[-1].id)
@dataclass
class _AgentAccessBootstrapCounts:
changed: int = 0
already_initialized: int = 0
@dataclass(frozen=True)
class _AgentAccessBootstrapOptions:
tenant_id: str
agent_id: str
creator_account_id: str | None
backing_app_id: str | None
operator_account_id: str
member_batch_size: int
apply: bool
initialized: bool
def _report_backing_app_specific_whitelist(options: _AgentAccessBootstrapOptions) -> None:
if not options.backing_app_id:
return
app_config = _resource_legacy_whitelist_config(
"app",
tenant_id=options.tenant_id,
operator_account_id=options.operator_account_id,
resource_id=options.backing_app_id,
)
if _normalize_rbac_whitelist_scope(app_config.rbac_whitelist_scope) is not RBACResourceWhitelistScope.SPECIFIC:
return
_emit_agent_migration_event(
{
"event": _AGENT_BACKING_APP_SPECIFIC_WHITELIST_EVENT,
"dry_run": not options.apply,
"tenant_id": options.tenant_id,
"agent_id": options.agent_id,
"app_id": options.backing_app_id,
"backing_app_account_ids": sorted(set(app_config.account_ids)),
}
)
def _write_agent_access_rows(options: _AgentAccessBootstrapOptions) -> None:
for batch in _workspace_member_account_id_batches(options.tenant_id, options.member_batch_size):
RBACService.AgentAccess.replace_user_access_policies(
tenant_id=options.tenant_id,
account_id=options.operator_account_id,
agent_id=options.agent_id,
target_account_id=None,
payload=ReplaceUserAccessPolicies(
access_policy_ids=[_RBAC_DEFAULT_ACCESS_POLICY_ID],
account_ids=batch,
),
)
if options.creator_account_id:
RBACService.AccessPolicies.sync_creator_access_policy_member_bindings(
tenant_id=options.tenant_id,
account_id=options.creator_account_id,
resource_type=RBACResourceType.AGENT,
resource_id=options.agent_id,
)
RBACService.AgentAccess.replace_whitelist(
tenant_id=options.tenant_id,
account_id=options.operator_account_id,
agent_id=options.agent_id,
payload=ReplaceMemberBindings(automatic_include_workspace_members=True),
)
def _emit_agent_access_bootstrap_skipped(
options: _AgentAccessBootstrapOptions,
reason: _AgentAccessBootstrapReason,
) -> None:
_emit_agent_migration_event(
{
"event": _AGENT_ACCESS_BOOTSTRAP_EVENT_KIND.skipped,
"reason": reason.value,
"dry_run": not options.apply,
"tenant_id": options.tenant_id,
"agent_id": options.agent_id,
"operator_account_id": options.operator_account_id,
}
)
def _agent_access_bootstrap_failure(
options: _AgentAccessBootstrapOptions,
exc: Exception,
) -> click.ClickException:
_emit_agent_migration_event(
{
"event": _AGENT_ACCESS_BOOTSTRAP_EVENT_KIND.failed,
"tenant_id": options.tenant_id,
"agent_id": options.agent_id,
"error": str(exc),
}
)
return click.ClickException(f"tenant {options.tenant_id} agent {options.agent_id}: {exc}")
def _bootstrap_agent_access(options: _AgentAccessBootstrapOptions, counts: _AgentAccessBootstrapCounts) -> None:
if options.initialized:
counts.already_initialized += 1
_emit_agent_access_bootstrap_skipped(options, _AgentAccessBootstrapReason.ALREADY_INITIALIZED)
return
config = RBACService.AgentAccess.legacy_whitelist_config(
tenant_id=options.tenant_id,
account_id=options.operator_account_id,
agent_id=options.agent_id,
)
try:
scope = _normalize_rbac_whitelist_scope(config.rbac_whitelist_scope)
except ValueError as exc:
raise _agent_access_bootstrap_failure(options, exc) from exc
if scope is None:
_emit_agent_access_bootstrap_skipped(options, _AgentAccessBootstrapReason.MISSING_WHITELIST_SCOPE)
return
account_ids = sorted(set(config.account_ids))
_report_backing_app_specific_whitelist(options)
if options.apply:
try:
_write_agent_access_rows(options)
except Exception as exc:
raise _agent_access_bootstrap_failure(options, exc) from exc
counts.changed += 1
event: dict[str, object] = {
"event": _AGENT_ACCESS_BOOTSTRAP_EVENT_KIND.outcome(apply=options.apply),
"dry_run": not options.apply,
"tenant_id": options.tenant_id,
"agent_id": options.agent_id,
"operator_account_id": options.operator_account_id,
"before": {"rbac_whitelist_scope": scope.value, "whitelist_account_ids": account_ids},
"after": {
"automatic_include_workspace_members": True,
"default_policy_member_source": _AGENT_ACCESS_BOOTSTRAP_MEMBER_SOURCE,
"creator_access_policy_synced": bool(options.creator_account_id),
},
}
if not options.creator_account_id:
event["reason"] = _AgentAccessBootstrapReason.NO_CREATOR.value
_emit_agent_migration_event(event)
def _bootstrap_tenant_agent_access(
tenant_id: str,
*,
agent_batch_size: int,
member_batch_size: int,
apply: bool,
counts: _AgentAccessBootstrapCounts,
) -> None:
owner_account_id: str | None = None
for rows in _iter_agent_row_batches(tenant_id, agent_batch_size):
initialized_agent_ids = set(
RBACService.Migrations.list_configured_agent_ids(tenant_id, [agent_id for agent_id, _, _ in rows])
)
for agent_id, creator_account_id, backing_app_id in rows:
if creator_account_id:
operator_account_id = creator_account_id
else:
if owner_account_id is None:
with session_factory.create_session() as session:
owner_account_id = _owner_account_id(tenant_id, session=session)
operator_account_id = owner_account_id
_bootstrap_agent_access(
_AgentAccessBootstrapOptions(
tenant_id=tenant_id,
agent_id=agent_id,
creator_account_id=creator_account_id,
backing_app_id=backing_app_id,
operator_account_id=operator_account_id,
member_batch_size=member_batch_size,
apply=apply,
initialized=agent_id in initialized_agent_ids,
),
counts,
)
@click.command(
"rbac-migrate-agent-permissions",
help=(
"Upgrade step for agent RBAC. Phase 1 asks the RBAC service to replace agent.manage on every "
"custom role with agent.create plus the agent.full_access binding. Phase 2 bootstraps the access "
"rows pre-existing agents never got, so they stay visible to workspace members. Agents that "
"already have a whitelist scope row are skipped. Dry run by default."
),
)
@click.option("--tenant-id", help="Only migrate a single workspace.")
@click.option(
"--batch-size",
default=500,
show_default=True,
type=click.IntRange(min=1),
help="Tenants fetched per database batch.",
)
@click.option(
"--agent-batch-size",
default=500,
show_default=True,
type=click.IntRange(min=1),
help="Agents fetched per database batch.",
)
@click.option(
"--member-batch-size",
default=_RBAC_RESOURCE_ACCESS_POLICY_BATCH_SIZE,
show_default=True,
type=click.IntRange(min=1),
help="Workspace members written per default-policy call when bootstrapping an agent.",
)
@click.option("--apply", is_flag=True, default=False, help="Write changes. Without it nothing is written.")
def migrate_agent_permissions_to_rbac(
tenant_id: str | None,
batch_size: int,
agent_batch_size: int,
member_batch_size: int,
apply: bool,
) -> None:
click.echo(click.style("Starting agent RBAC migration: custom roles holding agent.manage.", fg="green"))
tenant_count = 0
role_count = 0
skipped_count = 0
seen_template_ids: set[str] = set()
template_count = 0
agent_counts = _AgentAccessBootstrapCounts()
for workspace_id in _iter_tenant_ids(tenant_id, batch_size=batch_size):
tenant_count += 1
try:
report = RBACService.Migrations.migrate_agent_manage_roles(workspace_id, apply=apply)
except Exception as exc:
raise click.ClickException(f"tenant {workspace_id}: {exc}") from exc
for entry in report.roles:
role_count += 1
if entry.skipped:
skipped_count += 1
_emit_agent_migration_event(
_agent_manage_role_event(workspace_id, entry, apply=apply, kind=_AGENT_ROLE_MIGRATION_EVENT_KIND)
)
for template in report.role_templates:
if template.role_id in seen_template_ids:
continue
seen_template_ids.add(template.role_id)
template_count += 1
_emit_agent_migration_event(
_agent_manage_role_event(None, template, apply=apply, kind=_AGENT_ROLE_TEMPLATE_MIGRATION_EVENT_KIND)
)
_bootstrap_tenant_agent_access(
workspace_id,
agent_batch_size=agent_batch_size,
member_batch_size=member_batch_size,
apply=apply,
counts=agent_counts,
)
role_state = _AGENT_MIGRATION_STATE_LABEL[apply]
click.echo(
f"{tenant_count} tenant(s), {role_count} role(s) {role_state}, {skipped_count} skipped, "
f"{template_count} template(s) {role_state}, "
f"{agent_counts.changed} agent(s) {role_state}, "
f"{agent_counts.already_initialized} already initialised"
)
if not apply:
click.echo(click.style("Dry run: no changes written. Re-run with --apply.", fg="yellow"))

View File

@@ -10,8 +10,6 @@ import click
import sqlalchemy as sa
from sqlalchemy.orm import Session, sessionmaker
from configs import dify_config
from enums import CloudPlan, DeploymentEdition
from extensions.ext_database import db
from libs.datetime_utils import naive_utc_now
from services.clear_free_plan_tenant_expired_logs import ClearFreePlanTenantExpiredLogs
@@ -128,12 +126,14 @@ def _get_archive_candidate_tenant_ids_by_prefix(
def _filter_paid_workflow_archive_tenant_ids(tenant_ids: list[str]) -> tuple[list[str], list[str]]:
from configs import dify_config
from enums.cloud_plan import CloudPlan
from services.billing_service import BillingService
tenant_ids = sorted(set(tenant_ids))
if not tenant_ids:
return [], []
if dify_config.DEPLOYMENT_EDITION != DeploymentEdition.CLOUD:
if not dify_config.BILLING_ENABLED:
return tenant_ids, []
plans = BillingService.get_plan_bulk_with_cache(tenant_ids)
@@ -1462,7 +1462,7 @@ def cleanup_orphaned_draft_variables(
"--graceful-period",
default=21,
show_default=True,
help="Graceful period in days after subscription expiration; ignored outside the Cloud edition.",
help="Graceful period in days after subscription expiration, will be ignored when billing is disabled.",
)
@click.option("--dry-run", is_flag=True, default=False, help="Show messages logs would be cleaned without deleting")
def clean_expired_messages(
@@ -1514,8 +1514,8 @@ def clean_expired_messages(
if from_days_ago <= before_days:
raise click.UsageError("--from-days-ago must be greater than --before-days.")
# Create the policy for the configured deployment edition.
# NOTE: graceful_period is ignored outside the Cloud edition.
# Create policy based on billing configuration
# NOTE: graceful_period will be ignored when billing is disabled.
policy = create_message_clean_policy(graceful_period_days=graceful_period)
if from_days_ago is not None and before_days is not None:

View File

@@ -6,7 +6,7 @@ from sqlalchemy import delete, select, update
from sqlalchemy.orm import sessionmaker
from configs import dify_config
from enums import DeploymentEdition
from enums.deployment_edition import DeploymentEdition
from events.app_event import app_was_created
from extensions.ext_database import db
from extensions.ext_redis import redis_client

View File

@@ -2,10 +2,10 @@ import logging
from pathlib import Path
from typing import Any, override
from pydantic import model_validator
from pydantic.fields import FieldInfo
from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict, TomlConfigSettingsSource
from enums.deployment_edition import DeploymentEdition
from libs.file_utils import search_file_upwards
from .deploy import DeploymentConfig
@@ -92,13 +92,6 @@ class DifyConfig(
# for better readability and maintainability.
# Thanks for your concentration and consideration.
@model_validator(mode="after")
def validate_knowledge_fs_production_origin(self) -> "DifyConfig":
if self.DEPLOY_ENV.strip().upper() == "PRODUCTION":
if self.KNOWLEDGE_FS_BASE_URL and not self._is_secure_or_loopback_origin(self.KNOWLEDGE_FS_BASE_URL):
raise ValueError("KNOWLEDGE_FS_BASE_URL must use HTTPS in production unless it targets loopback")
return self
@classmethod
@override
def settings_customise_sources(
@@ -124,3 +117,11 @@ class DifyConfig(
),
),
)
@property
def DEPLOYMENT_EDITION(self) -> DeploymentEdition:
if self.EDITION == "CLOUD":
return DeploymentEdition.CLOUD
if self.ENTERPRISE_ENABLED:
return DeploymentEdition.ENTERPRISE
return DeploymentEdition.COMMUNITY

View File

@@ -1,8 +1,8 @@
from typing import Literal
from pydantic import Field
from pydantic_settings import BaseSettings
from enums import DeploymentEdition
class DeploymentConfig(BaseSettings):
"""
@@ -25,14 +25,9 @@ class DeploymentConfig(BaseSettings):
default=False,
)
DEPLOYMENT_EDITION: DeploymentEdition = Field(
description="Product edition of the application.",
default=DeploymentEdition.COMMUNITY,
)
INIT_PASSWORD: str = Field(
description="Password required before initializing a self-hosted deployment",
default="",
EDITION: Literal["SELF_HOSTED", "CLOUD"] = Field(
description="Deployment edition of the application (e.g., 'SELF_HOSTED', 'CLOUD')",
default="SELF_HOSTED",
)
DEPLOY_ENV: str = Field(

View File

@@ -8,6 +8,12 @@ class EnterpriseFeatureConfig(BaseSettings):
**Before using, please contact business@dify.ai by email to inquire about licensing matters.**
"""
ENTERPRISE_ENABLED: bool = Field(
description="Enable or disable enterprise-level features."
"Before using, please contact business@dify.ai by email to inquire about licensing matters.",
default=False,
)
WEBAPP_PUBLIC_ACCESS_ENABLED: bool = Field(
description="Whether admins are allowed to set a webapp's access mode to public (anyone with the link, "
"no auth). Disable in security-sensitive on-prem deployments.",
@@ -53,7 +59,7 @@ class EnterpriseTelemetryConfig(BaseSettings):
"""
ENTERPRISE_TELEMETRY_ENABLED: bool = Field(
description="Enable enterprise telemetry collection for enterprise deployments.",
description="Enable enterprise telemetry collection (also requires ENTERPRISE_ENABLED=true).",
default=False,
)
@@ -83,6 +89,11 @@ class EnterpriseTelemetryConfig(BaseSettings):
default=False,
)
ENTERPRISE_SERVICE_NAME: str = Field(
description="Service name for OTEL resource.",
default="dify",
)
ENTERPRISE_OTEL_SAMPLING_RATE: float = Field(
description="Sampling rate for enterprise traces (0.0 to 1.0, default 1.0 = 100%).",
default=1.0,

View File

@@ -1,10 +1,8 @@
from configs.extra.agent_backend_config import AgentBackendConfig
from configs.extra.archive_config import ArchiveStorageConfig
from configs.extra.knowledge_fs_config import KnowledgeFSConfig
from configs.extra.logstore_config import LogStoreConfig
from configs.extra.notion_config import NotionConfig
from configs.extra.sentry_config import SentryConfig
from configs.extra.turnstile_config import TurnstileConfig
class ExtraServiceConfig(
@@ -12,9 +10,7 @@ class ExtraServiceConfig(
AgentBackendConfig,
ArchiveStorageConfig,
KnowledgeFSConfig,
LogStoreConfig,
NotionConfig,
SentryConfig,
TurnstileConfig,
):
pass

View File

@@ -13,7 +13,7 @@ class AgentBackendConfig(BaseSettings):
)
AGENT_BACKEND_API_TOKEN: str | None = Field(
description="Bearer token for authenticating with the Agent backend control-plane API.",
description="Bearer token for authenticating with the Agent backend /runs API.",
default=None,
)
@@ -37,17 +37,9 @@ class AgentBackendConfig(BaseSettings):
default=3,
)
AGENT_BACKEND_HOME_SNAPSHOT_TIMEOUT_SECONDS: PositiveFloat = Field(
description=(
"Client timeout for Agent backend calls that may carry a Home Snapshot transfer: "
"snapshot capture and delete, and Execution Binding creation that restores one. "
),
default=45.0,
)
AGENT_BACKEND_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS: PositiveFloat = Field(
description="Client timeout for converting a Binding file to a ToolFile through the Agent backend.",
default=240,
AGENT_BACKEND_RUN_TIMEOUT_SECONDS: PositiveFloat = Field(
description="Total deadline for one Agent backend run event stream.",
default=1200,
)
AGENT_SHELL_ENABLED: bool = Field(

View File

@@ -1,82 +1,30 @@
"""Configuration for the optional KnowledgeFS control-plane integration."""
"""Configuration for the optional KnowledgeFS Console bridge."""
from ipaddress import ip_address
from urllib.parse import urlsplit
from pydantic import Field, PositiveFloat, PositiveInt, SecretStr, field_validator, model_validator
from pydantic import Field, PositiveFloat, SecretStr, field_validator, model_validator
from pydantic_settings import BaseSettings
class KnowledgeFSConfig(BaseSettings):
"""Server-only KnowledgeFS connection and rollout settings."""
KNOWLEDGE_FS_BACKGROUND_WORKER_ENABLED: bool = Field(
default=False,
description="Publish KnowledgeFS background work to dedicated Celery queues after worker rollout.",
)
KNOWLEDGE_FS_BACKGROUND_ENGINE_PATH: str = Field(default="/app/knowledge-fs/celery-worker.mjs")
KNOWLEDGE_FS_BACKGROUND_TASK_TIMEOUT_SECONDS: PositiveInt = Field(default=7200, ge=300, le=14400)
KNOWLEDGE_FS_BACKGROUND_POLL_INTERVAL_SECONDS: PositiveInt = Field(default=5, ge=1, le=60)
"""Server-only settings for the KnowledgeFS production connection."""
KNOWLEDGE_FS_ENABLED: bool = Field(
default=False,
description="Enable the KnowledgeFS control-plane product routes.",
description="Enable the private KnowledgeFS Console bridge.",
)
KNOWLEDGE_FS_COMMUNITY_DEV_ENABLED: bool = Field(
default=False,
description="Explicitly expose KnowledgeFS in Community edition for development deployments.",
)
KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED: bool = Field(
default=False,
description="Enable delivery of durable KnowledgeFS lifecycle commands after every rollout gate is ready.",
)
KNOWLEDGE_FS_INTEGRATED_PROVISION_READY: bool = Field(
default=False,
description="Confirm that the Capability-v2 integrated provision route is deployed and verified.",
)
KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY: bool = Field(
default=False,
description="Confirm that legacy KFS ACL mutation is frozen for integrated mode.",
)
KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS: PositiveInt = Field(default=15, le=300)
KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS: PositiveInt = Field(default=60, le=600)
KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE: PositiveInt = Field(default=25, le=1_000)
KNOWLEDGE_FS_BASE_URL: str | None = Field(default=None, description="KnowledgeFS gateway base URL.")
KNOWLEDGE_FS_CAPABILITY_V2_ENABLED: bool = Field(
default=False,
description="Prepare resource-scoped Capability v2 issuance; disabled until rollout approval.",
)
KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID: str | None = Field(
KNOWLEDGE_FS_JWT_SECRET: SecretStr | None = Field(
default=None,
description="Identifier for the current asymmetric Capability v2 signing key.",
min_length=32,
description="Shared secret used to sign short-lived KnowledgeFS service JWTs.",
)
KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM: SecretStr | None = Field(
default=None,
description="Server-only PEM for the current Capability v2 RSA signing key.",
)
KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS: str | None = Field(
default=None,
description="Optional public-only JWKS JSON retained during key rotation overlap.",
)
KNOWLEDGE_FS_CAPABILITY_V2_ISSUER: str = Field(default="dify-control-plane", min_length=1)
KNOWLEDGE_FS_CAPABILITY_V2_AUDIENCE: str = Field(default="knowledge-fs", min_length=1)
KNOWLEDGE_FS_CAPABILITY_V2_MAX_TTL_SECONDS: PositiveInt = Field(default=60, le=60)
KNOWLEDGE_FS_JWKS_CACHE_MAX_AGE_SECONDS: PositiveInt = Field(default=300, le=86_400)
KNOWLEDGE_FS_PRODUCT_MAX_RESPONSE_BYTES: PositiveInt = Field(default=4 * 1024 * 1024, le=16 * 1024 * 1024)
KNOWLEDGE_FS_SSE_READ_TIMEOUT_SECONDS: PositiveFloat = Field(default=300.0, le=3600.0, allow_inf_nan=False)
KNOWLEDGE_FS_TIMEOUT_SECONDS: PositiveFloat = Field(default=10.0, le=60.0, allow_inf_nan=False)
KNOWLEDGE_FS_RETRIEVAL_TEST_TIMEOUT_SECONDS: float = Field(
default=75.0,
gt=60.0,
le=120.0,
allow_inf_nan=False,
description="Timeout for retrieval tests and Agent investigation triage, which have a 60-second model budget.",
)
@field_validator(
"KNOWLEDGE_FS_BASE_URL",
"KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID",
"KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM",
"KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS",
"KNOWLEDGE_FS_JWT_SECRET",
mode="before",
)
@classmethod
@@ -92,45 +40,25 @@ class KnowledgeFSConfig(BaseSettings):
@field_validator("KNOWLEDGE_FS_BASE_URL")
@classmethod
def validate_base_url(cls, value: str | None) -> str | None:
return cls._validate_origin(value, name="KNOWLEDGE_FS_BASE_URL")
@classmethod
def _validate_origin(cls, value: str | None, *, name: str) -> str | None:
if value is None:
return None
parsed = urlsplit(value)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise ValueError(f"{name} must be an absolute HTTP(S) URL")
raise ValueError("KNOWLEDGE_FS_BASE_URL must be an absolute HTTP(S) URL")
try:
_ = parsed.port
except ValueError as exc:
raise ValueError(f"{name} must include a valid port") from exc
if parsed.username or parsed.password or parsed.query or parsed.fragment or parsed.path not in {"", "/"}:
raise ValueError(f"{name} must be an origin without credentials, path, query, or fragment")
raise ValueError("KNOWLEDGE_FS_BASE_URL must include a valid port") from exc
if parsed.username or parsed.password or parsed.query or parsed.fragment:
raise ValueError("KNOWLEDGE_FS_BASE_URL must not include credentials, query, or fragment")
return value.rstrip("/")
@model_validator(mode="after")
def validate_enabled_connection(self) -> "KnowledgeFSConfig":
if self.KNOWLEDGE_FS_ENABLED:
if not self.KNOWLEDGE_FS_BASE_URL:
raise ValueError("KnowledgeFS base URL is required when the integration is enabled")
if not self.KNOWLEDGE_FS_CAPABILITY_V2_ENABLED:
raise ValueError("KnowledgeFS product routes require Capability v2 when enabled")
if self.KNOWLEDGE_FS_CAPABILITY_V2_ENABLED and not (
self.KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID and self.KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM
):
raise ValueError("Capability v2 signing kid and private key are required when issuance is enabled")
if not self.KNOWLEDGE_FS_ENABLED:
return self
if bool(self.KNOWLEDGE_FS_BASE_URL) != bool(self.KNOWLEDGE_FS_JWT_SECRET):
raise ValueError("KNOWLEDGE_FS_BASE_URL and KNOWLEDGE_FS_JWT_SECRET must be configured together")
if not self.KNOWLEDGE_FS_BASE_URL:
raise ValueError("KnowledgeFS connection settings are required when the integration is enabled")
return self
@staticmethod
def _is_secure_or_loopback_origin(value: str) -> bool:
parsed = urlsplit(value)
if parsed.scheme == "https":
return True
hostname = (parsed.hostname or "").rstrip(".").lower()
if hostname == "localhost":
return True
try:
return ip_address(hostname).is_loopback
except ValueError:
return False

View File

@@ -1,11 +0,0 @@
from pydantic_settings import BaseSettings
class LogStoreConfig(BaseSettings):
"""Migration controls for repositories backed by Aliyun LogStore."""
LOGSTORE_DUAL_WRITE_ENABLED: bool = False
# Keep workflow graphs in LogStore by default. Deployments may disable this
# while migrating large graph payloads to another persistence owner.
LOGSTORE_ENABLE_PUT_GRAPH_FIELD: bool = True

View File

@@ -1,41 +0,0 @@
from pydantic import Field, SecretStr, field_validator
from pydantic_settings import BaseSettings
class TurnstileConfig(BaseSettings):
"""Server-side Cloudflare Turnstile settings for Cloud sign-in."""
TURNSTILE_SECRET_KEY: SecretStr | None = Field(
default=None,
description="Secret key used to validate Cloudflare Turnstile tokens.",
)
TURNSTILE_ALLOWED_HOSTNAMES: str = Field(
default="",
description="Comma-separated parent or exact hostnames accepted from Turnstile.",
)
TURNSTILE_EMAIL_CODE_VERIFY_REQUIRED: bool = Field(
default=False,
description=(
"Require a separate Turnstile challenge when verifying email login codes on Dify Cloud. "
"Enable after the compatible web client has been deployed."
),
)
@field_validator("TURNSTILE_SECRET_KEY", mode="before")
@classmethod
def normalize_secret_key(cls, value: object) -> object:
if isinstance(value, SecretStr):
normalized = value.get_secret_value().strip()
return SecretStr(normalized) if normalized else None
if isinstance(value, str):
normalized = value.strip()
return normalized or None
return value
@property
def TURNSTILE_ALLOWED_HOSTNAME_SET(self) -> frozenset[str]:
return frozenset(
hostname.strip().lower().strip(".")
for hostname in self.TURNSTILE_ALLOWED_HOSTNAMES.split(",")
if hostname.strip().strip(".")
)

Some files were not shown because too many files have changed in this diff Show More