refactor(datasources): migrate catalog consumers to generated queries (#42713)

This commit is contained in:
yyh
2026-09-22 05:55:00 +00:00
committed by GitHub
parent 67ab356116
commit 26d4b9ead8
121 changed files with 3052 additions and 2305 deletions

View File

@@ -45,6 +45,7 @@ from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpErr
from core.app.apps.base_app_queue_manager import AppQueueManager
from core.app.apps.pipeline.pipeline_generator import PipelineGenerator
from core.app.entities.app_invoke_entities import InvokeFrom
from core.plugin.entities.plugin_daemon import PluginDatasourceProviderEntity
from extensions.ext_database import db
from factories import variable_factory
from fields.base import ResponseModel
@@ -57,7 +58,6 @@ from fields.workflow_run_fields import (
workflow_run_pagination_response_source,
workflow_run_response_source,
)
from graphon.model_runtime.utils.encoders import jsonable_encoder
from libs import helper
from libs.helper import TimestampField, UUIDStrOrEmpty, dump_response
from libs.login import login_required
@@ -151,6 +151,14 @@ class RagPipelineOpaqueResponse(RootModel[Any]):
root: Any
class RagPipelineDatasourceProviderResponse(PluginDatasourceProviderEntity, ResponseModel):
pass
class RagPipelineDatasourceListResponse(RootModel[list[RagPipelineDatasourceProviderResponse]]):
pass
class RagPipelineStepParametersResponse(ResponseModel):
variables: Any
@@ -175,6 +183,8 @@ register_response_schema_models(
console_ns,
DefaultBlockConfigResponse,
DefaultBlockConfigsResponse,
RagPipelineDatasourceListResponse,
RagPipelineDatasourceProviderResponse,
RagPipelineOpaqueResponse,
RagPipelineStepParametersResponse,
RagPipelineWorkflowPublishResponse,
@@ -1000,13 +1010,14 @@ class RagPipelineWorkflowRunNodeExecutionListApi(Resource):
@console_ns.route("/rag/pipelines/datasource-plugins")
class DatasourceListApi(Resource):
@console_ns.response(200, "Success", console_ns.models[RagPipelineOpaqueResponse.__name__])
@console_ns.response(200, "Success", console_ns.models[RagPipelineDatasourceListResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@with_current_tenant_id
def get(self, current_tenant_id: str):
return jsonable_encoder(RagPipelineManageService.list_rag_pipeline_datasources(current_tenant_id))
providers = RagPipelineManageService.list_rag_pipeline_datasources(current_tenant_id)
return RagPipelineDatasourceListResponse.model_validate(providers, from_attributes=True).model_dump(mode="json")
@console_ns.route("/rag/pipelines/<uuid:pipeline_id>/workflows/draft/nodes/<string:node_id>/last-run")

View File

@@ -7671,7 +7671,7 @@ Update account-level Step-by-step Tour state
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Success | **application/json**: [RagPipelineOpaqueResponse](#ragpipelineopaqueresponse)<br> |
| 200 | Success | **application/json**: [RagPipelineDatasourceListResponse](#ragpipelinedatasourcelistresponse)<br> |
### [POST] /rag/pipelines/imports
#### Request Body
@@ -17561,6 +17561,25 @@ Model class for provider custom model configuration.
| ---- | ---- | ----------- | -------- |
| id | string | | Yes |
#### DatasourceEntity
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| description | [I18nObject](#i18nobject) | The label of the datasource | Yes |
| identity | [DatasourceIdentity](#datasourceidentity) | | Yes |
| output_schema | object | | No |
| parameters | [ [DatasourceParameter](#datasourceparameter) ] | | No |
#### DatasourceIdentity
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| author | string | The author of the datasource | Yes |
| icon | string | | No |
| label | [I18nObject](#i18nobject) | The label of the datasource | Yes |
| name | string | The name of the datasource | Yes |
| provider | string | The provider of the datasource | Yes |
#### DatasourceNodeRunPayload
| Name | Type | Description | Required |
@@ -17596,6 +17615,35 @@ Model class for provider custom model configuration.
| oauth_custom_client_params | object | Masked plugin-defined OAuth client parameters, when configured for the tenant. | Yes |
| redirect_uri | string | | Yes |
#### DatasourceParameter
Overrides type
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| auto_generate | [PluginParameterAutoGenerate](#pluginparameterautogenerate) | | No |
| default | number<br>integer<br>string<br>boolean<br>[ object ]<br>object | | No |
| description | [I18nObject](#i18nobject) | The description of the parameter | Yes |
| label | [I18nObject](#i18nobject) | The label presented to the user | Yes |
| max | number<br>integer | | No |
| min | number<br>integer | | No |
| name | string | The name of the parameter | Yes |
| options | [ [PluginParameterOption](#pluginparameteroption) ] | | No |
| placeholder | [I18nObject](#i18nobject) | The placeholder presented to the user | No |
| precision | integer | | No |
| required | boolean | | No |
| scope | string | | No |
| template | [PluginParameterTemplate](#pluginparametertemplate) | | No |
| type | [DatasourceParameterType](#datasourceparametertype) | The type of the parameter | Yes |
#### DatasourceParameterType
removes TOOLS_SELECTOR from PluginParameterType
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| DatasourceParameterType | string | removes TOOLS_SELECTOR from PluginParameterType | |
#### DatasourceProviderAuthListResponse
| Name | Type | Description | Required |
@@ -17629,6 +17677,16 @@ Datasource provider entity
| oauth_schema | [OAuthSchema](#oauthschema) | | No |
| provider_type | [DatasourceProviderType](#datasourceprovidertype) | | Yes |
#### DatasourceProviderEntityWithPlugin
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| credentials_schema | [ [ProviderConfig](#providerconfig) ] | | No |
| datasources | [ [DatasourceEntity](#datasourceentity) ] | | No |
| identity | [DatasourceProviderIdentity](#datasourceprovideridentity) | | Yes |
| oauth_schema | [OAuthSchema](#oauthschema) | | No |
| provider_type | [DatasourceProviderType](#datasourceprovidertype) | | Yes |
#### DatasourceProviderIdentity
| Name | Type | Description | Required |
@@ -21526,6 +21584,22 @@ Resource types understood by access policies.
| ---- | ---- | ----------- | -------- |
| yaml_content | string | | Yes |
#### RagPipelineDatasourceListResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| RagPipelineDatasourceListResponse | array | | |
#### RagPipelineDatasourceProviderResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| declaration | [DatasourceProviderEntityWithPlugin](#datasourceproviderentitywithplugin) | | Yes |
| is_authorized | boolean | | No |
| plugin_id | string | | Yes |
| plugin_unique_identifier | string | | Yes |
| provider | string | | Yes |
#### RagPipelineImportCheckDependenciesResponse
| Name | Type | Description | Required |

View File

@@ -0,0 +1,194 @@
from inspect import unwrap
from unittest.mock import patch
import pytest
from flask import Flask
from pydantic import ValidationError
from controllers.console.datasets.rag_pipeline.rag_pipeline_workflow import (
DatasourceListApi,
RagPipelineDatasourceListResponse,
RagPipelineDatasourceProviderResponse,
)
from core.datasource.entities.datasource_entities import DatasourceParameter
from core.plugin.entities import OAuthSchema
from core.plugin.entities.plugin_daemon import PluginDatasourceProviderEntity
from core.plugin.impl.datasource import PluginDatasourceManager
from services.datasource_provider_service import DatasourceProviderService
from services.rag_pipeline.rag_pipeline_manage_service import RagPipelineManageService
def _provider() -> PluginDatasourceProviderEntity:
return PluginDatasourceProviderEntity.model_validate(
{
"provider": "documents",
"plugin_id": "example/documents",
"plugin_unique_identifier": "example/documents:0.0.1@checksum",
"declaration": {
"identity": {
"author": "example",
"name": "documents",
"label": {"en_US": "Documents"},
"description": {"en_US": "Import documents"},
"icon": "icon.svg",
"tags": None,
},
"provider_type": "online_document",
"credentials_schema": [{"type": "secret-input", "name": "token", "required": True}],
"datasources": [
{
"identity": {
"author": "example",
"name": "pages",
"provider": "documents",
"label": {"en_US": "Pages"},
},
"description": {"en_US": "Import pages"},
"parameters": [
{
"name": "limit",
"type": "number",
"label": {"en_US": "Limit"},
"description": {"en_US": "Maximum pages"},
"required": True,
"default": 0,
"min": 0,
"max": 100,
},
{
"name": "archived",
"type": "boolean",
"label": {"en_US": "Archived"},
"description": {"en_US": "Include archived pages"},
"required": True,
"default": False,
},
],
"output_schema": {
"type": "object",
"properties": {"page": {"anyOf": [{"type": "string"}, {"type": "null"}]}},
"additionalProperties": False,
},
}
],
},
}
)
@pytest.mark.parametrize("credentials", [None, {"token": "secret"}])
def test_catalog_serializes_real_plugin_and_local_file_providers(
app: Flask, credentials: dict[str, str] | None
) -> None:
provider = _provider()
api = DatasourceListApi()
with (
app.test_request_context("/rag/pipelines/datasource-plugins"),
patch.object(
PluginDatasourceManager, "_request_with_plugin_daemon_response", return_value=[provider]
) as request,
patch.object(
DatasourceProviderService, "get_datasource_credentials", return_value=credentials
) as get_credentials,
):
result = unwrap(api.get)(api, "tenant-1")
local_file, remote = result
assert local_file["provider"] == "file"
assert local_file["declaration"]["provider_type"] == "local_file"
assert local_file["declaration"]["identity"]["name"] == "langgenius/file/file"
assert local_file["is_authorized"] is True
assert local_file["declaration"]["datasources"][0]["identity"]["provider"] == "langgenius/file/file"
assert local_file["declaration"]["datasources"][0]["output_schema"] is None
assert remote == provider.model_dump(mode="json")
assert remote["provider"] == "documents"
assert remote["plugin_unique_identifier"] == "example/documents:0.0.1@checksum"
assert remote["is_authorized"] is bool(credentials)
identity = remote["declaration"]["identity"]
assert identity["name"] == "example/documents/documents"
assert identity["icon"].endswith("/console/api/workspaces/current/plugin/icon?tenant_id=tenant-1&filename=icon.svg")
assert identity["tags"] is None
assert identity["label"]["pt_BR"] == "Documents"
datasource = remote["declaration"]["datasources"][0]
assert datasource["identity"]["provider"] == "example/documents/documents"
assert datasource["identity"]["icon"] is None
assert datasource["parameters"][0]["default"] == 0
assert type(datasource["parameters"][0]["default"]) is int
assert datasource["parameters"][0]["min"] == 0
assert datasource["parameters"][0]["max"] == 100
assert datasource["parameters"][1]["default"] is False
assert datasource["output_schema"]["properties"]["page"] == {"anyOf": [{"type": "string"}, {"type": "null"}]}
assert datasource["output_schema"]["additionalProperties"] is False
request.assert_called_once()
assert request.call_args.args[:3] == (
"GET",
"plugin/tenant-1/management/datasources",
list[PluginDatasourceProviderEntity],
)
get_credentials.assert_called_once_with(tenant_id="tenant-1", provider="documents", plugin_id="example/documents")
def test_catalog_keeps_oauth_only_provider_unauthorized_without_credentials(app: Flask) -> None:
provider = _provider()
provider.declaration.credentials_schema = []
provider.declaration.oauth_schema = OAuthSchema()
api = DatasourceListApi()
with (
app.test_request_context("/rag/pipelines/datasource-plugins"),
patch.object(PluginDatasourceManager, "fetch_datasource_providers", return_value=[provider]),
patch.object(DatasourceProviderService, "get_datasource_credentials", return_value=None) as get_credentials,
):
result = unwrap(api.get)(api, "tenant-1")
assert result[0]["is_authorized"] is False
assert result[0]["declaration"]["oauth_schema"] == {"client_schema": [], "credentials_schema": []}
get_credentials.assert_called_once_with(tenant_id="tenant-1", provider="documents", plugin_id="example/documents")
def test_catalog_keeps_provider_visible_after_credential_failure(app: Flask) -> None:
provider = _provider()
api = DatasourceListApi()
with (
app.test_request_context("/rag/pipelines/datasource-plugins"),
patch.object(PluginDatasourceManager, "fetch_datasource_providers", return_value=[provider]),
patch.object(DatasourceProviderService, "get_datasource_credentials", side_effect=ValueError("decrypt failed")),
):
result = unwrap(api.get)(api, "tenant-1")
assert result == [provider.model_dump(mode="json")]
assert result[0]["is_authorized"] is False
def test_catalog_serializes_empty_list(app: Flask) -> None:
api = DatasourceListApi()
with (
app.test_request_context("/rag/pipelines/datasource-plugins"),
patch.object(RagPipelineManageService, "list_rag_pipeline_datasources", return_value=[]),
):
result = unwrap(api.get)(api, "tenant-1")
assert result == []
def test_catalog_list_schema_reuses_datasource_domain_entities() -> None:
schema = RagPipelineDatasourceListResponse.model_json_schema(mode="serialization")
assert schema["type"] == "array"
assert schema["items"] == {"$ref": "#/$defs/RagPipelineDatasourceProviderResponse"}
declaration = schema["$defs"]["RagPipelineDatasourceProviderResponse"]["properties"]["declaration"]
assert declaration == {"$ref": "#/$defs/DatasourceProviderEntityWithPlugin"}
assert schema["$defs"]["DatasourceParameterType"]["enum"] == [
value.value for value in DatasourceParameter.DatasourceParameterType
]
def test_catalog_response_rejects_tool_only_parameter_types() -> None:
payload = _provider().model_dump(mode="json")
payload["declaration"]["datasources"][0]["parameters"][0]["type"] = "dynamic-select"
with pytest.raises(ValidationError, match="string.*number.*boolean"):
RagPipelineDatasourceProviderResponse.model_validate(payload)

View File

@@ -866,6 +866,53 @@ def test_console_installed_plugin_ids_exported_schema_is_lightweight(tmp_path: P
}
def test_console_datasource_catalog_exports_domain_response_schema(tmp_path: Path) -> None:
from dev.generate_swagger_specs import generate_specs
written_paths = generate_specs(tmp_path)
console_openapi_path = next(path for path in written_paths if path.name == "console-openapi.json")
payload = json.loads(console_openapi_path.read_text(encoding="utf-8"))
operation = payload["paths"]["/rag/pipelines/datasource-plugins"]["get"]
assert operation.get("parameters", []) == []
assert "requestBody" not in operation
assert operation["responses"]["200"]["content"]["application/json"]["schema"] == {
"$ref": "#/components/schemas/RagPipelineDatasourceListResponse"
}
schemas = payload["components"]["schemas"]
assert schemas["RagPipelineDatasourceListResponse"]["type"] == "array"
assert schemas["RagPipelineDatasourceListResponse"]["items"] == {
"$ref": "#/components/schemas/RagPipelineDatasourceProviderResponse"
}
provider = schemas["RagPipelineDatasourceProviderResponse"]
assert set(provider["properties"]) == {
"provider",
"plugin_unique_identifier",
"plugin_id",
"is_authorized",
"declaration",
}
assert provider["properties"]["declaration"] == {"$ref": "#/components/schemas/DatasourceProviderEntityWithPlugin"}
assert set(schemas["DatasourceParameterType"]["enum"]) == {
"string",
"number",
"boolean",
"select",
"secret-input",
"file",
"files",
"system-files",
}
assert schemas["DatasourceEntity"]["properties"]["output_schema"]["anyOf"] == [
{"additionalProperties": True, "type": "object"},
{"type": "null"},
]
recommended = payload["paths"]["/rag/pipelines/recommended-plugins"]["get"]
assert recommended["responses"]["200"]["content"]["application/json"]["schema"] == {
"$ref": "#/components/schemas/RagPipelineOpaqueResponse"
}
def test_console_model_provider_summary_exported_schema_is_lightweight(tmp_path: Path):
from dev.generate_swagger_specs import generate_specs

View File

@@ -2552,11 +2552,6 @@
"count": 8
}
},
"web/app/components/plugins/plugin-detail-panel/datasource-action-list.tsx": {
"typescript/no-explicit-any": {
"count": 1
}
},
"web/app/components/plugins/plugin-detail-panel/detail-header.tsx": {
"no-barrel-files/no-barrel-files": {
"count": 1
@@ -3487,26 +3482,11 @@
"count": 1
}
},
"web/app/components/workflow/nodes/data-source/default.ts": {
"typescript/no-explicit-any": {
"count": 5
}
},
"web/app/components/workflow/nodes/data-source/hooks/use-before-run-form.ts": {
"typescript/no-explicit-any": {
"count": 1
}
},
"web/app/components/workflow/nodes/data-source/hooks/use-config.ts": {
"typescript/no-explicit-any": {
"count": 6
}
},
"web/app/components/workflow/nodes/data-source/panel.tsx": {
"typescript/no-explicit-any": {
"count": 3
}
},
"web/app/components/workflow/nodes/data-source/types.ts": {
"typescript/no-explicit-any": {
"count": 1
@@ -4451,11 +4431,6 @@
"count": 3
}
},
"web/app/components/workflow/utils/data-source.ts": {
"typescript/no-explicit-any": {
"count": 1
}
},
"web/app/components/workflow/utils/index.ts": {
"no-barrel-files/no-barrel-files": {
"count": 10
@@ -5007,7 +4982,7 @@
},
"web/service/use-pipeline.ts": {
"@tanstack/query/prefer-query-options": {
"count": 9
"count": 8
},
"no-restricted-imports": {
"count": 1

View File

@@ -89,7 +89,7 @@ export type PipelineTemplateDetailResponse = {
name: string
}
export type RagPipelineOpaqueResponse = unknown
export type RagPipelineDatasourceListResponse = Array<RagPipelineDatasourceProviderResponse>
export type RagPipelineImportPayload = {
description?: string | null
@@ -107,6 +107,8 @@ export type RagPipelineImportCheckDependenciesResponse = {
leaked_dependencies?: Array<PluginDependency>
}
export type RagPipelineOpaqueResponse = unknown
export type WorkflowRunPaginationResponse = {
data: Array<WorkflowRunForListResponse>
has_more: boolean
@@ -373,6 +375,14 @@ export type PipelineTemplateItemResponse = {
privacy_policy?: string | null
}
export type RagPipelineDatasourceProviderResponse = {
declaration: DatasourceProviderEntityWithPlugin
is_authorized?: boolean
plugin_id: string
plugin_unique_identifier: string
provider: string
}
export type PluginDependency = {
current_identifier?: string | null
type: PluginDependencyType
@@ -493,6 +503,14 @@ export type DatasetWeightedScoreResponse = {
weight_type?: string | null
}
export type DatasourceProviderEntityWithPlugin = {
credentials_schema?: Array<ProviderConfig>
datasources?: Array<DatasourceEntity>
identity: DatasourceProviderIdentity
oauth_schema?: OAuthSchema | null
provider_type: DatasourceProviderType
}
export type PluginDependencyType = 'github' | 'marketplace' | 'package'
export type Github = {
@@ -522,6 +540,169 @@ export type DatasetVectorSettingResponse = {
vector_weight?: number | null
}
export type ProviderConfig = {
default?: number | string | number | boolean | null
help?: I18nObject | null
label?: I18nObject | null
multiple?: boolean
name: string
options?: Array<Option> | null
placeholder?: I18nObject | null
required?: boolean
scope?: AppSelectorScope | ModelSelectorScope | ToolSelectorScope | null
type: ProviderConfigType
url?: string | null
}
export type DatasourceEntity = {
description: I18nObject
identity: DatasourceIdentity
output_schema?: {
[key: string]: unknown
} | null
parameters?: Array<DatasourceParameter>
}
export type DatasourceProviderIdentity = {
author: string
description: CoreToolsEntitiesCommonEntitiesI18nObject
icon: string
label: CoreToolsEntitiesCommonEntitiesI18nObject
name: string
tags?: Array<ToolLabelEnum> | null
}
export type OAuthSchema = {
client_schema?: Array<ProviderConfig>
credentials_schema?: Array<ProviderConfig>
}
export type DatasourceProviderType =
| 'local_file'
| 'online_document'
| 'online_drive'
| 'website_crawl'
export type I18nObject = {
en_US: string
ja_JP?: string | null
pt_BR?: string | null
zh_Hans?: string | null
}
export type Option = {
label: I18nObject
value: string
}
export type AppSelectorScope = 'all' | 'chat' | 'completion' | 'workflow'
export type ModelSelectorScope =
| 'llm'
| 'moderation'
| 'rerank'
| 'speech2text'
| 'text-embedding'
| 'tts'
| 'vision'
export type ToolSelectorScope = 'all' | 'builtin' | 'custom' | 'workflow'
export type ProviderConfigType =
| 'app-selector'
| 'array[tools]'
| 'boolean'
| 'model-selector'
| 'secret-input'
| 'select'
| 'text-input'
export type DatasourceIdentity = {
author: string
icon?: string | null
label: I18nObject
name: string
provider: string
}
export type DatasourceParameter = {
auto_generate?: PluginParameterAutoGenerate | null
default?:
| number
| number
| string
| boolean
| Array<unknown>
| {
[key: string]: unknown
}
| null
description: I18nObject
label: I18nObject
max?: number | number | null
min?: number | number | null
name: string
options?: Array<PluginParameterOption>
placeholder?: I18nObject | null
precision?: number | null
required?: boolean
scope?: string | null
template?: PluginParameterTemplate | null
type: DatasourceParameterType
}
export type CoreToolsEntitiesCommonEntitiesI18nObject = {
en_US: string
ja_JP?: string | null
pt_BR?: string | null
zh_Hans?: string | null
}
export type ToolLabelEnum =
| 'business'
| 'design'
| 'education'
| 'entertainment'
| 'finance'
| 'image'
| 'medical'
| 'news'
| 'other'
| 'productivity'
| 'rag'
| 'search'
| 'social'
| 'travel'
| 'utilities'
| 'videos'
| 'weather'
export type PluginParameterAutoGenerate = {
type: PluginParameterAutoGenerateType
}
export type PluginParameterOption = {
icon?: string | null
label: I18nObject
value: string
}
export type PluginParameterTemplate = {
enabled?: boolean
}
export type DatasourceParameterType =
| 'boolean'
| 'file'
| 'files'
| 'number'
| 'secret-input'
| 'select'
| 'string'
| 'system-files'
export type PluginParameterAutoGenerateType = 'prompt_instruction'
export type DeleteRagPipelineCustomizedTemplatesByTemplateIdData = {
body?: never
path: {
@@ -649,7 +830,7 @@ export type GetRagPipelinesDatasourcePluginsData = {
}
export type GetRagPipelinesDatasourcePluginsResponses = {
200: RagPipelineOpaqueResponse
200: RagPipelineDatasourceListResponse
}
export type GetRagPipelinesDatasourcePluginsResponse =

View File

@@ -39,11 +39,6 @@ export const zPipelineTemplateDetailResponse = z.object({
name: z.string(),
})
/**
* RagPipelineOpaqueResponse
*/
export const zRagPipelineOpaqueResponse = z.unknown()
/**
* RagPipelineImportPayload
*/
@@ -59,6 +54,11 @@ export const zRagPipelineImportPayload = z.object({
yaml_url: z.string().nullish(),
})
/**
* RagPipelineOpaqueResponse
*/
export const zRagPipelineOpaqueResponse = z.unknown()
/**
* SimpleResultResponse
*/
@@ -666,6 +666,273 @@ export const zDatasetDetailResponse = z.object({
word_count: z.int(),
})
/**
* DatasourceProviderType
*
* Enum class for datasource provider
*/
export const zDatasourceProviderType = z.enum([
'local_file',
'online_document',
'online_drive',
'website_crawl',
])
/**
* I18nObject
*
* Model class for i18n object.
*/
export const zI18nObject = z.object({
en_US: z.string(),
ja_JP: z.string().nullish(),
pt_BR: z.string().nullish(),
zh_Hans: z.string().nullish(),
})
/**
* Option
*/
export const zOption = z.object({
label: zI18nObject,
value: z.string(),
})
/**
* AppSelectorScope
*/
export const zAppSelectorScope = z.enum(['all', 'chat', 'completion', 'workflow'])
/**
* ModelSelectorScope
*/
export const zModelSelectorScope = z.enum([
'llm',
'moderation',
'rerank',
'speech2text',
'text-embedding',
'tts',
'vision',
])
/**
* ToolSelectorScope
*/
export const zToolSelectorScope = z.enum(['all', 'builtin', 'custom', 'workflow'])
/**
* ProviderConfigType
*/
export const zProviderConfigType = z.enum([
'app-selector',
'array[tools]',
'boolean',
'model-selector',
'secret-input',
'select',
'text-input',
])
/**
* ProviderConfig
*
* Model class for common provider settings like credentials
*/
export const zProviderConfig = z.object({
default: z.union([z.int(), z.string(), z.number(), z.boolean()]).nullish(),
help: zI18nObject.nullish(),
label: zI18nObject.nullish(),
multiple: z.boolean().optional().default(false),
name: z.string(),
options: z.array(zOption).nullish(),
placeholder: zI18nObject.nullish(),
required: z.boolean().optional().default(false),
scope: z.union([zAppSelectorScope, zModelSelectorScope, zToolSelectorScope]).nullish(),
type: zProviderConfigType,
url: z.string().nullish(),
})
/**
* OAuthSchema
*
* OAuth schema
*/
export const zOAuthSchema = z.object({
client_schema: z.array(zProviderConfig).optional(),
credentials_schema: z.array(zProviderConfig).optional(),
})
/**
* DatasourceIdentity
*/
export const zDatasourceIdentity = z.object({
author: z.string(),
icon: z.string().nullish(),
label: zI18nObject,
name: z.string(),
provider: z.string(),
})
/**
* I18nObject
*
* Model class for i18n object.
*/
export const zCoreToolsEntitiesCommonEntitiesI18nObject = z.object({
en_US: z.string(),
ja_JP: z.string().nullish(),
pt_BR: z.string().nullish(),
zh_Hans: z.string().nullish(),
})
/**
* ToolLabelEnum
*/
export const zToolLabelEnum = z.enum([
'business',
'design',
'education',
'entertainment',
'finance',
'image',
'medical',
'news',
'other',
'productivity',
'rag',
'search',
'social',
'travel',
'utilities',
'videos',
'weather',
])
/**
* DatasourceProviderIdentity
*/
export const zDatasourceProviderIdentity = z.object({
author: z.string(),
description: zCoreToolsEntitiesCommonEntitiesI18nObject,
icon: z.string(),
label: zCoreToolsEntitiesCommonEntitiesI18nObject,
name: z.string(),
tags: z.array(zToolLabelEnum).nullish().default([]),
})
/**
* PluginParameterOption
*/
export const zPluginParameterOption = z.object({
icon: z.string().nullish(),
label: zI18nObject,
value: z.string(),
})
/**
* PluginParameterTemplate
*/
export const zPluginParameterTemplate = z.object({
enabled: z.boolean().optional().default(false),
})
/**
* DatasourceParameterType
*
* removes TOOLS_SELECTOR from PluginParameterType
*/
export const zDatasourceParameterType = z.enum([
'boolean',
'file',
'files',
'number',
'secret-input',
'select',
'string',
'system-files',
])
/**
* PluginParameterAutoGenerateType
*/
export const zPluginParameterAutoGenerateType = z.enum(['prompt_instruction'])
/**
* PluginParameterAutoGenerate
*/
export const zPluginParameterAutoGenerate = z.object({
type: zPluginParameterAutoGenerateType,
})
/**
* DatasourceParameter
*
* Overrides type
*/
export const zDatasourceParameter = z.object({
auto_generate: zPluginParameterAutoGenerate.nullish(),
default: z
.union([
z.number(),
z.int(),
z.string(),
z.boolean(),
z.array(z.unknown()),
z.record(z.string(), z.unknown()),
])
.nullish(),
description: zI18nObject,
label: zI18nObject,
max: z.union([z.number(), z.int()]).nullish(),
min: z.union([z.number(), z.int()]).nullish(),
name: z.string(),
options: z.array(zPluginParameterOption).optional(),
placeholder: zI18nObject.nullish(),
precision: z.int().nullish(),
required: z.boolean().optional().default(false),
scope: z.string().nullish(),
template: zPluginParameterTemplate.nullish(),
type: zDatasourceParameterType,
})
/**
* DatasourceEntity
*/
export const zDatasourceEntity = z.object({
description: zI18nObject,
identity: zDatasourceIdentity,
output_schema: z.record(z.string(), z.unknown()).nullish(),
parameters: z.array(zDatasourceParameter).optional(),
})
/**
* DatasourceProviderEntityWithPlugin
*/
export const zDatasourceProviderEntityWithPlugin = z.object({
credentials_schema: z.array(zProviderConfig).optional(),
datasources: z.array(zDatasourceEntity).optional(),
identity: zDatasourceProviderIdentity,
oauth_schema: zOAuthSchema.nullish(),
provider_type: zDatasourceProviderType,
})
/**
* RagPipelineDatasourceProviderResponse
*/
export const zRagPipelineDatasourceProviderResponse = z.object({
declaration: zDatasourceProviderEntityWithPlugin,
is_authorized: z.boolean().optional().default(false),
plugin_id: z.string(),
plugin_unique_identifier: z.string(),
provider: z.string(),
})
/**
* RagPipelineDatasourceListResponse
*/
export const zRagPipelineDatasourceListResponse = z.array(zRagPipelineDatasourceProviderResponse)
export const zDeleteRagPipelineCustomizedTemplatesByTemplateIdPath = z.object({
template_id: z.string(),
})
@@ -734,7 +1001,7 @@ export const zGetRagPipelineTemplatesByTemplateIdResponse = zPipelineTemplateDet
/**
* Success
*/
export const zGetRagPipelinesDatasourcePluginsResponse = zRagPipelineOpaqueResponse
export const zGetRagPipelinesDatasourcePluginsResponse = zRagPipelineDatasourceListResponse
export const zPostRagPipelinesImportsBody = zRagPipelineImportPayload

View File

@@ -0,0 +1,72 @@
import type { UnsupportedNode } from '@dify/contracts/enterprise-app-deploy/types.gen'
import type { PropsWithChildren } from 'react'
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
import { renderHook, waitFor } from '@testing-library/react'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { BlockEnum } from '@/app/components/workflow/types'
import { consoleQuery } from '@/service/console'
import { useGetProviderIcon } from '../use-provider-icon'
const request = vi.hoisted(() => vi.fn<(url: string) => Promise<Response>>())
vi.mock('@/service/base', () => ({ request }))
vi.mock('@/service/use-tools', () => ({
useAllBuiltInTools: () => ({}),
useAllCustomTools: () => ({}),
useAllWorkflowTools: () => ({}),
useAllMCPTools: () => ({}),
}))
vi.mock('@/service/use-triggers', () => ({ useAllTriggerPlugins: () => ({}) }))
vi.mock('@/hooks/use-theme', () => ({ default: () => ({ theme: 'light' }) }))
vi.mock('@/utils/var', async (importOriginal) => ({
...(await importOriginal<typeof import('@/utils/var')>()),
basePath: '/base',
}))
const node: UnsupportedNode = {
id: 'source-1',
type: BlockEnum.DataSource,
title: 'Local files',
provider: {
plugin_id: 'langgenius/file',
provider_id: 'file',
provider_name: 'file',
provider_type: 'local_file',
},
}
describe('deployment datasource icons', () => {
let client: QueryClient
const wrapper = ({ children }: PropsWithChildren) => (
<QueryClientProvider client={client}>{children}</QueryClientProvider>
)
beforeEach(() => {
vi.clearAllMocks()
client = new QueryClient({ defaultOptions: { queries: { retry: false } } })
request.mockResolvedValue(Response.json([createDatasourceProvider()]))
})
afterEach(() => client.clear())
it('admits the catalog only when a datasource node needs its icon', async () => {
const { result, rerender } = renderHook(({ nodes }) => useGetProviderIcon(nodes), {
initialProps: { nodes: [] as UnsupportedNode[] },
wrapper,
})
expect(request).not.toHaveBeenCalled()
rerender({ nodes: [node] })
await waitFor(() => expect(result.current(node)).toBe('/base/datasource.svg'))
expect(request).toHaveBeenCalledOnce()
expect(
client.getQueryData(consoleQuery.rag.pipelines.datasourcePlugins.get.queryKey()),
).toEqual([createDatasourceProvider()])
})
it('returns no icon when the deployment node has no provider metadata', () => {
const unknown: UnsupportedNode = { id: 'unknown', type: 'human-input', title: 'Input' }
const { result } = renderHook(() => useGetProviderIcon([unknown]), { wrapper })
expect(result.current(unknown)).toBeUndefined()
expect(request).not.toHaveBeenCalled()
})
})

View File

@@ -1,11 +1,11 @@
import type { UnsupportedNode } from '@dify/contracts/enterprise-app-deploy/types.gen'
import type { Node } from '@/app/components/workflow/types'
import { useCallback, useMemo } from 'react'
import { transformDataSourceToTool } from '@/app/components/workflow/block-selector/utils'
import { useQuery } from '@tanstack/react-query'
import { useCallback } from 'react'
import { findNodeIcon } from '@/app/components/workflow/hooks/use-tool-icon'
import { BlockEnum } from '@/app/components/workflow/types'
import useTheme from '@/hooks/use-theme'
import { useDataSourceList } from '@/service/use-pipeline'
import { consoleQuery } from '@/service/console'
import {
useAllBuiltInTools,
useAllCustomTools,
@@ -39,10 +39,13 @@ export const useGetProviderIcon = (nodes: UnsupportedNode[]) => {
const { data: customTools } = useAllCustomTools(hasToolNode)
const { data: workflowTools } = useAllWorkflowTools(hasToolNode)
const { data: mcpTools } = useAllMCPTools(hasToolNode)
const { data: dataSources } = useDataSourceList(hasDataSourceNode)
const { data: dataSourceList } = useQuery(
consoleQuery.rag.pipelines.datasourcePlugins.get.queryOptions({
enabled: hasDataSourceNode,
}),
)
const { data: triggerPlugins } = useAllTriggerPlugins(hasTriggerPluginNode)
const { theme } = useTheme()
const dataSourceList = useMemo(() => dataSources?.map(transformDataSourceToTool), [dataSources])
return useCallback(
(node: UnsupportedNode) => {

View File

@@ -1,148 +0,0 @@
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import { renderHook } from '@testing-library/react'
import { beforeEach, describe, expect, it, vi } from 'vite-plus/test'
import { useDatasourceIcon } from '../hooks'
const mockTransformDataSourceToTool = vi.fn()
vi.mock('@/app/components/workflow/block-selector/utils', () => ({
transformDataSourceToTool: (...args: unknown[]) => mockTransformDataSourceToTool(...args),
}))
let mockDataSourceListReturn: {
data:
| Array<{
plugin_id: string
provider: string
declaration: { identity: { icon: string; author: string } }
}>
| undefined
isSuccess: boolean
}
vi.mock('@/service/use-pipeline', () => ({
useDataSourceList: () => mockDataSourceListReturn,
}))
vi.mock('@/utils/var', () => ({
basePath: '',
}))
const createMockDataSourceNode = (overrides?: Partial<DataSourceNodeType>): DataSourceNodeType =>
({
plugin_id: 'plugin-abc',
provider_type: 'builtin',
provider_name: 'web-scraper',
datasource_name: 'scraper',
datasource_label: 'Web Scraper',
datasource_parameters: {},
datasource_configurations: {},
title: 'DataSource',
desc: '',
type: '' as DataSourceNodeType['type'],
...overrides,
}) as DataSourceNodeType
describe('useDatasourceIcon', () => {
beforeEach(() => {
vi.clearAllMocks()
mockDataSourceListReturn = { data: undefined, isSuccess: false }
mockTransformDataSourceToTool.mockReset()
})
// Returns undefined when data has not loaded
describe('Loading State', () => {
it('should return undefined when data is not loaded (isSuccess false)', () => {
mockDataSourceListReturn = { data: undefined, isSuccess: false }
const { result } = renderHook(() => useDatasourceIcon(createMockDataSourceNode()))
expect(result.current).toBeUndefined()
})
})
// Returns correct icon when plugin_id matches
describe('Icon Resolution', () => {
it('should return correct icon when plugin_id matches', () => {
const mockIcon = 'https://example.com/icon.svg'
mockDataSourceListReturn = {
data: [
{
plugin_id: 'plugin-abc',
provider: 'web-scraper',
declaration: { identity: { icon: mockIcon, author: 'dify' } },
},
],
isSuccess: true,
}
mockTransformDataSourceToTool.mockImplementation(
(item: { plugin_id: string; declaration: { identity: { icon: string } } }) => ({
plugin_id: item.plugin_id,
icon: item.declaration.identity.icon,
}),
)
const { result } = renderHook(() =>
useDatasourceIcon(createMockDataSourceNode({ plugin_id: 'plugin-abc' })),
)
expect(result.current).toBe(mockIcon)
})
it('should return undefined when plugin_id does not match', () => {
mockDataSourceListReturn = {
data: [
{
plugin_id: 'plugin-xyz',
provider: 'other',
declaration: { identity: { icon: '/icon.svg', author: 'dify' } },
},
],
isSuccess: true,
}
mockTransformDataSourceToTool.mockImplementation(
(item: { plugin_id: string; declaration: { identity: { icon: string } } }) => ({
plugin_id: item.plugin_id,
icon: item.declaration.identity.icon,
}),
)
const { result } = renderHook(() =>
useDatasourceIcon(createMockDataSourceNode({ plugin_id: 'plugin-abc' })),
)
expect(result.current).toBeUndefined()
})
})
// basePath prepending
describe('basePath Prepending', () => {
it('should prepend basePath to icon URL when not already included', () => {
// basePath is mocked as '' so prepending '' to '/icon.png' results in '/icon.png'
// The important thing is that the forEach logic runs without error
mockDataSourceListReturn = {
data: [
{
plugin_id: 'plugin-abc',
provider: 'web-scraper',
declaration: { identity: { icon: '/icon.png', author: 'dify' } },
},
],
isSuccess: true,
}
mockTransformDataSourceToTool.mockImplementation(
(item: { plugin_id: string; declaration: { identity: { icon: string } } }) => ({
plugin_id: item.plugin_id,
icon: item.declaration.identity.icon,
}),
)
const { result } = renderHook(() =>
useDatasourceIcon(createMockDataSourceNode({ plugin_id: 'plugin-abc' })),
)
// With empty basePath, icon stays as '/icon.png'
expect(result.current).toBe('/icon.png')
})
})
})

View File

@@ -1,86 +1,122 @@
import type { PropsWithChildren } from 'react'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import { fireEvent, render, screen } from '@testing-library/react'
import { beforeEach, describe, expect, it, vi } from 'vite-plus/test'
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
import { render, screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { BlockEnum } from '@/app/components/workflow/types'
import { consoleQuery } from '@/service/console'
import OptionCard from '../option-card'
const TEST_ICON_URL = 'https://example.com/test-icon.png'
vi.mock('../hooks', () => ({
useDatasourceIcon: () => TEST_ICON_URL,
const request = vi.hoisted(() => vi.fn<(url: string) => Promise<Response>>())
vi.mock('@/service/base', () => ({ request }))
vi.mock('@/utils/var', async (importOriginal) => ({
...(await importOriginal<typeof import('@/utils/var')>()),
basePath: '/base',
}))
vi.mock('../datasource-icon', () => ({
default: ({ iconUrl }: { iconUrl: string }) => (
<img data-testid="datasource-icon" src={iconUrl} alt="datasource" />
),
}))
const createMockNodeData = (overrides: Partial<DataSourceNodeType> = {}): DataSourceNodeType =>
({
title: 'Test Node',
desc: '',
type: {} as DataSourceNodeType['type'],
plugin_id: 'test-plugin',
provider_type: 'builtin',
provider_name: 'test-provider',
datasource_name: 'test-ds',
datasource_label: 'Test DS',
datasource_parameters: {},
datasource_configurations: {},
...overrides,
}) as DataSourceNodeType
const node: DataSourceNodeType = {
plugin_id: 'langgenius/file',
provider_type: 'local_file',
provider_name: 'file',
datasource_name: 'local-file',
datasource_label: 'Local File',
datasource_parameters: {},
datasource_configurations: {},
title: 'DataSource',
desc: '',
type: BlockEnum.DataSource,
}
describe('OptionCard', () => {
const defaultProps = {
label: 'Google Drive',
selected: false,
nodeData: createMockNodeData(),
onClick: vi.fn(),
}
let client: QueryClient
const key = consoleQuery.rag.pipelines.datasourcePlugins.get.queryKey()
const wrapper = ({ children }: PropsWithChildren) => (
<QueryClientProvider client={client}>{children}</QueryClientProvider>
)
beforeEach(() => {
vi.clearAllMocks()
client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: Infinity } } })
request.mockResolvedValue(Response.json([]))
})
afterEach(() => client.clear())
it.each([
['/icon.png', '/base/icon.png'],
['/base/icon.png', '/base/icon.png'],
['https://example.com/icon.svg', 'https://example.com/icon.svg'],
['//example.com/icon.svg', '//example.com/icon.svg'],
])('renders %s without changing the shared response', async (icon, expected) => {
const provider = createDatasourceProvider()
provider.declaration.identity.icon = icon
request.mockResolvedValue(Response.json([provider]))
const { container } = render(<OptionCard label="Local File" selected nodeData={node} />, {
wrapper,
})
await waitFor(() =>
expect(container.querySelector('.bg-cover')).toHaveStyle({
backgroundImage: `url("${expected}")`,
}),
)
expect(client.getQueryData(key)).toEqual([provider])
expect(request).toHaveBeenCalledOnce()
})
// Rendering: label text and icon
describe('Rendering', () => {
it('should render label text', () => {
render(<OptionCard {...defaultProps} />)
it('renders a cached icon while revalidating the catalog', () => {
client.setQueryData(key, [createDatasourceProvider()])
request.mockReturnValue(new Promise(() => {}))
expect(screen.getByText('Google Drive')).toBeInTheDocument()
const { container } = render(<OptionCard label="Local File" selected nodeData={node} />, {
wrapper,
})
it('should render datasource icon with correct URL', () => {
render(<OptionCard {...defaultProps} />)
const icon = screen.getByTestId('datasource-icon')
expect(icon).toHaveAttribute('src', TEST_ICON_URL)
})
it('should set title attribute on label element', () => {
render(<OptionCard {...defaultProps} />)
expect(screen.getByTitle('Google Drive')).toBeInTheDocument()
expect(container.querySelector('.bg-cover')).toHaveStyle({
backgroundImage: 'url("/base/datasource.svg")',
})
})
// User interactions: clicking the card
describe('User Interactions', () => {
it('should call onClick when clicked', () => {
render(<OptionCard {...defaultProps} />)
it('keeps a missing or pending provider icon empty instead of constructing a URL', async () => {
const { container } = render(
<OptionCard label="Local File" selected={false} nodeData={node} />,
{ wrapper },
)
fireEvent.click(screen.getByText('Google Drive'))
expect(defaultProps.onClick).toHaveBeenCalledOnce()
})
it('should not throw when onClick is undefined', () => {
expect(() => {
const { container } = render(<OptionCard {...defaultProps} onClick={undefined} />)
fireEvent.click(container.firstElementChild!)
}).not.toThrow()
})
expect(container.querySelector<HTMLElement>('.bg-cover')?.style.backgroundImage).toBe('')
await waitFor(() => expect(client.getQueryState(key)?.status).toBe('success'))
expect(container.querySelector<HTMLElement>('.bg-cover')?.style.backgroundImage).toBe('')
expect(screen.getByText('Local File')).toBeInTheDocument()
})
// Props: selected state applies different styles
it('resolves an older saved provider without plugin metadata', async () => {
const provider = createDatasourceProvider()
request.mockResolvedValue(Response.json([provider]))
const { container } = render(
<OptionCard label="Local File" selected={false} nodeData={{ ...node, plugin_id: '' }} />,
{ wrapper },
)
await waitFor(() =>
expect(container.querySelector('.bg-cover')).toHaveStyle({
backgroundImage: 'url("/base/datasource.svg")',
}),
)
expect(client.getQueryData(key)).toEqual([provider])
})
it('keeps the label disclosure and selection callback', async () => {
const user = userEvent.setup()
const onClick = vi.fn()
render(<OptionCard label="Local File" selected={false} nodeData={node} onClick={onClick} />, {
wrapper,
})
await user.click(screen.getByText('Local File'))
expect(onClick).toHaveBeenCalledOnce()
expect(screen.getByTitle('Local File')).toBeInTheDocument()
})
})

View File

@@ -5,7 +5,7 @@ import { memo } from 'react'
type DatasourceIconProps = {
size?: string
className?: string
iconUrl: string
iconUrl?: string
}
const ICON_CONTAINER_CLASSNAME_SIZE_MAP: Record<string, string> = {
@@ -26,7 +26,7 @@ const DatasourceIcon: FC<DatasourceIconProps> = ({ size = 'sm', className, iconU
<div
className="size-full shrink-0 rounded-md bg-cover bg-center"
style={{
backgroundImage: `url(${iconUrl})`,
backgroundImage: iconUrl ? `url(${iconUrl})` : undefined,
}}
/>
</div>

View File

@@ -1,25 +0,0 @@
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import { useMemo } from 'react'
import { transformDataSourceToTool } from '@/app/components/workflow/block-selector/utils'
import { useDataSourceList } from '@/service/use-pipeline'
import { basePath } from '@/utils/var'
export const useDatasourceIcon = (data: DataSourceNodeType) => {
const { data: dataSourceListData, isSuccess } = useDataSourceList(true)
const datasourceIcon = useMemo(() => {
if (!isSuccess) return
const dataSourceList = [...(dataSourceListData || [])]
dataSourceList.forEach((item) => {
const icon = item.declaration.identity.icon
if (typeof icon == 'string' && !icon.includes(basePath))
item.declaration.identity.icon = `${basePath}${icon}`
})
const formattedDataSourceList = dataSourceList.map((item) => transformDataSourceToTool(item))
return formattedDataSourceList?.find(
(toolWithProvider) => toolWithProvider.plugin_id === data.plugin_id,
)?.icon
}, [data.plugin_id, dataSourceListData, isSuccess])
return datasourceIcon
}

View File

@@ -1,8 +1,11 @@
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import { cn } from '@langgenius/dify-ui/cn'
import { useQuery } from '@tanstack/react-query'
import * as React from 'react'
import { resolveDatasourceIcon } from '@/app/components/rag-pipeline/utils/datasource-icon'
import { matchDataSource } from '@/app/components/workflow/utils/plugin-install-check'
import { consoleQuery } from '@/service/console'
import DatasourceIcon from './datasource-icon'
import { useDatasourceIcon } from './hooks'
type OptionCardProps = {
label: string
@@ -12,7 +15,11 @@ type OptionCardProps = {
}
const OptionCard = ({ label, selected, nodeData, onClick }: OptionCardProps) => {
const iconUrl = useDatasourceIcon(nodeData) as string
const { data: dataSourceList } = useQuery(
consoleQuery.rag.pipelines.datasourcePlugins.get.queryOptions(),
)
const provider = dataSourceList && matchDataSource(dataSourceList, nodeData)
const iconUrl = provider && resolveDatasourceIcon(provider.declaration.identity.icon)
return (
<div

View File

@@ -4,6 +4,7 @@ import type { Node } from '@/app/components/workflow/types'
import { screen } from '@testing-library/react'
import { NuqsTestingAdapter } from 'nuqs/adapters/testing'
import { beforeEach, describe, expect, it, vi } from 'vite-plus/test'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { DatasourceType } from '@/models/pipeline'
import { renderWithConsoleQuery } from '@/test/console/query-data'
import StepOneContent from '../step-one-content'
@@ -169,6 +170,7 @@ vi.mock('@/hooks/use-theme', () => ({
// Mock upload service
vi.mock('@/service/base', () => ({
upload: vi.fn().mockResolvedValue({ id: 'uploaded-file-id' }),
request: vi.fn(async () => Response.json([createDatasourceProvider()])),
}))
vi.mock('@/next/navigation', () => ({
@@ -187,21 +189,6 @@ vi.mock('@/service/use-pipeline', () => ({
data: { pages: [] },
isLoading: false,
})),
useDataSourceList: vi.fn(() => ({
data: [
{
type: 'local_file',
declaration: {
identity: {
name: 'Local File',
icon: '/icons/local-file.svg',
},
},
},
],
isSuccess: true,
isLoading: false,
})),
useCrawlResult: vi.fn(() => ({
data: { data: [] },
isLoading: false,

View File

@@ -13,7 +13,6 @@ import {
useInvalidDataSourceListAuth,
useInvalidDefaultDataSourceListAuth,
} from '@/service/use-datasource'
import { useInvalidDataSourceList } from '@/service/use-pipeline'
import { render } from '@/test/console/render'
import Card from '../card'
import { useDataSourceAuthUpdate } from '../hooks'
@@ -111,10 +110,6 @@ vi.mock('../hooks', () => ({
useDataSourceAuthUpdate: vi.fn(),
}))
vi.mock('@/service/use-pipeline', () => ({
useInvalidDataSourceList: vi.fn(() => vi.fn()),
}))
type UsePluginAuthActionReturn = ReturnType<typeof usePluginAuthAction>
type UseGetDataSourceOAuthUrlReturn = ReturnType<typeof useGetDataSourceOAuthUrl>
type UseRenderI18nObjectReturn = ReturnType<typeof useRenderI18nObject>
@@ -124,12 +119,10 @@ describe('Card Component', () => {
const mockRenderI18nObjectResult = vi.fn((obj: Record<string, string>) => obj.en_US)
const mockInvalidateDataSourceListAuth = vi.fn()
const mockInvalidDefaultDataSourceListAuth = vi.fn()
const mockInvalidateDataSourceList = vi.fn()
const mockInvalidateDataSourceAuth = vi.fn()
const mockHandleAuthUpdate = vi.fn(() => {
mockInvalidateDataSourceListAuth()
mockInvalidDefaultDataSourceListAuth()
mockInvalidateDataSourceList()
mockInvalidateDataSourceAuth()
})
@@ -192,7 +185,6 @@ describe('Card Component', () => {
vi.mocked(useInvalidDefaultDataSourceListAuth).mockReturnValue(
mockInvalidDefaultDataSourceListAuth,
)
vi.mocked(useInvalidDataSourceList).mockReturnValue(mockInvalidateDataSourceList)
vi.mocked(useInvalidDataSourceAuth).mockReturnValue(mockInvalidateDataSourceAuth)
vi.mocked(usePluginAuthAction).mockReturnValue(mockPluginAuthActionReturn)
@@ -207,7 +199,6 @@ describe('Card Component', () => {
const expectAuthUpdated = () => {
expect(mockInvalidateDataSourceListAuth).toHaveBeenCalled()
expect(mockInvalidDefaultDataSourceListAuth).toHaveBeenCalled()
expect(mockInvalidateDataSourceList).toHaveBeenCalled()
expect(mockInvalidateDataSourceAuth).toHaveBeenCalled()
}

View File

@@ -7,12 +7,12 @@ import { usePluginsWithLatestVersion } from '@/app/components/plugins/hooks'
import { usePluginAuthAction } from '@/app/components/plugins/plugin-auth'
import { PluginCategoryEnum } from '@/app/components/plugins/types'
import { useRenderI18nObject } from '@/hooks/use-i18n'
import { consoleQuery } from '@/service/console'
import {
useGetDataSourceListAuth,
useGetDataSourceOAuthUrl,
useInvalidDataSourceListAuth,
} from '@/service/use-datasource'
import { useInvalidDataSourceList } from '@/service/use-pipeline'
import {
useCheckInstalled,
useInstalledPluginList,
@@ -42,10 +42,6 @@ vi.mock('@/service/use-datasource', () => ({
useInvalidDataSourceListAuth: vi.fn(),
}))
vi.mock('@/service/use-pipeline', () => ({
useInvalidDataSourceList: vi.fn(),
}))
vi.mock('@/service/use-plugins', () => ({
useCheckInstalled: vi.fn(),
useInstalledPluginList: vi.fn(),
@@ -57,8 +53,10 @@ vi.mock('@/app/components/plugins/hooks', () => ({
}))
vi.mock('../plugin-actions', () => ({
default: ({ detail }: { detail: { plugin_id: string } }) => (
<button data-testid={`plugin-actions-${detail.plugin_id}`}>Actions</button>
default: ({ detail, onUpdate }: { detail: { plugin_id: string }; onUpdate?: () => void }) => (
<button data-testid={`plugin-actions-${detail.plugin_id}`} onClick={onUpdate}>
Actions
</button>
),
}))
@@ -147,6 +145,7 @@ describe('DataSourcePage Component', () => {
agent_strategy: undefined,
trigger: undefined,
datasource: {
provider_type: 'online_document',
identity: {
author: 'Dify',
name: 'Dify Source',
@@ -184,7 +183,6 @@ describe('DataSourcePage Component', () => {
mutateAsync: vi.fn(),
} as unknown as ReturnType<typeof useGetDataSourceOAuthUrl>)
vi.mocked(useInvalidDataSourceListAuth).mockReturnValue(vi.fn())
vi.mocked(useInvalidDataSourceList).mockReturnValue(vi.fn())
vi.mocked(useInstalledPluginList).mockReturnValue({
data: { plugins: [], total: 0 },
} as unknown as ReturnType<typeof useInstalledPluginList>)
@@ -294,10 +292,15 @@ describe('DataSourcePage Component', () => {
} as unknown as ReturnType<typeof useInstalledPluginList>)
// Act
renderWithConsoleQuery(<DataSourcePage />, {
const { queryClient } = renderWithConsoleQuery(<DataSourcePage />, {
systemFeatures: { enable_marketplace: false },
})
const catalogKey = consoleQuery.rag.pipelines.datasourcePlugins.get.queryKey()
queryClient.setQueryData(catalogKey, [])
fireEvent.click(screen.getByTestId('plugin-actions-plugin-1'))
expect(queryClient.getQueryState(catalogKey)?.isInvalidated).toBe(true)
// Assert
expect(screen.getByTestId('plugin-actions-plugin-1')).toBeInTheDocument()
expect(useInstalledPluginList).toHaveBeenLastCalledWith({

View File

@@ -95,6 +95,7 @@ const createPluginDetail = (overrides: Partial<PluginDetail> = {}): PluginDetail
icon: 'icon.png',
tags: [],
datasource: {
provider_type: 'online_document',
identity: {
author: 'acme',
name: 'datasource-provider',

View File

@@ -1,88 +1,48 @@
import type { PropsWithChildren } from 'react'
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
import { act, renderHook } from '@testing-library/react'
import {
useInvalidDataSourceAuth,
useInvalidDataSourceListAuth,
useInvalidDefaultDataSourceListAuth,
} from '@/service/use-datasource'
import { useInvalidDataSourceList } from '@/service/use-pipeline'
import { createElement } from 'react'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test'
import { consoleQuery } from '@/service/console'
import { useDataSourceAuthUpdate } from '../use-data-source-auth-update'
/**
* useDataSourceAuthUpdate Hook Tests
* This hook manages the invalidation of various data source related queries.
*/
const mocks = vi.hoisted(() => ({ auth: vi.fn(), list: vi.fn(), defaults: vi.fn() }))
vi.mock('@/service/use-datasource', () => ({
useInvalidDataSourceAuth: vi.fn(),
useInvalidDataSourceListAuth: vi.fn(),
useInvalidDefaultDataSourceListAuth: vi.fn(),
useInvalidDataSourceAuth: () => mocks.auth,
useInvalidDataSourceListAuth: () => mocks.list,
useInvalidDefaultDataSourceListAuth: () => mocks.defaults,
}))
vi.mock('@/service/use-pipeline', () => ({
useInvalidDataSourceList: vi.fn(),
}))
describe('useDataSourceAuthUpdate', () => {
const mockInvalidateDataSourceAuth = vi.fn()
const mockInvalidateDataSourceListAuth = vi.fn()
const mockInvalidDefaultDataSourceListAuth = vi.fn()
const mockInvalidateDataSourceList = vi.fn()
describe('datasource authentication reconciliation', () => {
let client: QueryClient
const catalogKey = consoleQuery.rag.pipelines.datasourcePlugins.get.queryKey()
const unrelatedKey = consoleQuery.account.profile.get.key()
const wrapper = ({ children }: PropsWithChildren) =>
createElement(QueryClientProvider, { client }, children)
beforeEach(() => {
vi.clearAllMocks()
vi.mocked(useInvalidDataSourceAuth).mockReturnValue(mockInvalidateDataSourceAuth)
vi.mocked(useInvalidDataSourceListAuth).mockReturnValue(mockInvalidateDataSourceListAuth)
vi.mocked(useInvalidDefaultDataSourceListAuth).mockReturnValue(
mockInvalidDefaultDataSourceListAuth,
)
vi.mocked(useInvalidDataSourceList).mockReturnValue(mockInvalidateDataSourceList)
client = new QueryClient()
client.setQueryData(catalogKey, [])
client.setQueryData(unrelatedKey, { cached: true })
})
afterEach(() => client.clear())
describe('handleAuthUpdate', () => {
it('should call all invalidate functions when handleAuthUpdate is invoked', () => {
// Arrange
const pluginId = 'test-plugin-id'
const provider = 'test-provider'
const { result } = renderHook(() =>
it('invalidates catalog availability and retains authorization reconciliation', () => {
const { result } = renderHook(
() =>
useDataSourceAuthUpdate({
pluginId,
provider,
pluginId: 'langgenius/notion',
provider: 'notion',
}),
)
{ wrapper },
)
// Assert Initialization
expect(useInvalidDataSourceAuth).toHaveBeenCalledWith({ pluginId, provider })
act(() => result.current.handleAuthUpdate())
// Act
act(() => {
result.current.handleAuthUpdate()
})
// Assert Invalidation
expect(mockInvalidateDataSourceListAuth).toHaveBeenCalledTimes(1)
expect(mockInvalidDefaultDataSourceListAuth).toHaveBeenCalledTimes(1)
expect(mockInvalidateDataSourceList).toHaveBeenCalledTimes(1)
expect(mockInvalidateDataSourceAuth).toHaveBeenCalledTimes(1)
})
it('should maintain stable handleAuthUpdate reference if dependencies do not change', () => {
// Arrange
const props = {
pluginId: 'stable-plugin',
provider: 'stable-provider',
}
const { result, rerender } = renderHook(
({ pluginId, provider }) => useDataSourceAuthUpdate({ pluginId, provider }),
{ initialProps: props },
)
const firstHandleAuthUpdate = result.current.handleAuthUpdate
// Act
rerender(props)
// Assert
expect(result.current.handleAuthUpdate).toBe(firstHandleAuthUpdate)
})
expect(client.getQueryState(catalogKey)?.isInvalidated).toBe(true)
expect(client.getQueryState(unrelatedKey)?.isInvalidated).toBe(false)
expect(mocks.auth).toHaveBeenCalledOnce()
expect(mocks.list).toHaveBeenCalledOnce()
expect(mocks.defaults).toHaveBeenCalledOnce()
})
})

View File

@@ -1,10 +1,11 @@
import { useQueryClient } from '@tanstack/react-query'
import { useCallback } from 'react'
import { consoleQuery } from '@/service/console'
import {
useInvalidDataSourceAuth,
useInvalidDataSourceListAuth,
useInvalidDefaultDataSourceListAuth,
} from '@/service/use-datasource'
import { useInvalidDataSourceList } from '@/service/use-pipeline'
export const useDataSourceAuthUpdate = ({
pluginId,
@@ -13,9 +14,9 @@ export const useDataSourceAuthUpdate = ({
pluginId: string
provider: string
}) => {
const queryClient = useQueryClient()
const invalidateDataSourceListAuth = useInvalidDataSourceListAuth()
const invalidDefaultDataSourceListAuth = useInvalidDefaultDataSourceListAuth()
const invalidateDataSourceList = useInvalidDataSourceList()
const invalidateDataSourceAuth = useInvalidDataSourceAuth({
pluginId,
provider,
@@ -23,11 +24,13 @@ export const useDataSourceAuthUpdate = ({
const handleAuthUpdate = useCallback(() => {
invalidateDataSourceListAuth()
invalidDefaultDataSourceListAuth()
invalidateDataSourceList()
queryClient.invalidateQueries({
queryKey: consoleQuery.rag.pipelines.datasourcePlugins.get.key(),
})
invalidateDataSourceAuth()
}, [
invalidateDataSourceListAuth,
invalidateDataSourceList,
queryClient,
invalidateDataSourceAuth,
invalidDefaultDataSourceListAuth,
])

View File

@@ -1,5 +1,5 @@
import type { ReactNode } from 'react'
import { useSuspenseQuery } from '@tanstack/react-query'
import { useQueryClient, useSuspenseQuery } from '@tanstack/react-query'
import { memo, useCallback, useMemo, useState } from 'react'
import { Trans, useTranslation } from 'react-i18next'
import { SearchInput } from '@/app/components/base/search-input'
@@ -10,8 +10,8 @@ import { PluginCategoryEnum } from '@/app/components/plugins/types'
import { STEP_BY_STEP_TOUR_TARGETS } from '@/app/components/step-by-step-tour/target-registry'
import { systemFeaturesQueryOptions } from '@/features/system-features/client'
import { useRenderI18nObject } from '@/hooks/use-i18n'
import { consoleQuery } from '@/service/console'
import { useGetDataSourceListAuth, useInvalidDataSourceListAuth } from '@/service/use-datasource'
import { useInvalidDataSourceList } from '@/service/use-pipeline'
import { useInstalledPluginList, useInvalidateInstalledPluginList } from '@/service/use-plugins'
import UpdateSettingDialog from '../update-setting-dialog'
import Card from './card'
@@ -55,6 +55,7 @@ function DataSourceListSkeleton() {
const DataSourcePage = ({ layout, onOpenMarketplace, stickyToolbar }: DataSourcePageProps) => {
const { t } = useTranslation()
const queryClient = useQueryClient()
const renderI18nObject = useRenderI18nObject()
const [searchText, setSearchText] = useState('')
const { canSetPluginPreferences } = usePluginSettingsAccess()
@@ -69,7 +70,6 @@ const DataSourcePage = ({ layout, onOpenMarketplace, stickyToolbar }: DataSource
const pluginListWithLatestVersion = usePluginsWithLatestVersion(installedPluginList?.plugins)
const invalidateInstalledPluginList = useInvalidateInstalledPluginList()
const invalidateDataSourceListAuth = useInvalidDataSourceListAuth()
const invalidateDataSourceList = useInvalidDataSourceList()
const dataSources = useMemo(() => data?.result ?? [], [data?.result])
const dataSourcePluginDetails = useMemo(() => {
return pluginListWithLatestVersion.filter(
@@ -97,8 +97,10 @@ const DataSourcePage = ({ layout, onOpenMarketplace, stickyToolbar }: DataSource
const handlePluginUpdate = useCallback(() => {
invalidateInstalledPluginList()
invalidateDataSourceListAuth()
invalidateDataSourceList()
}, [invalidateDataSourceList, invalidateDataSourceListAuth, invalidateInstalledPluginList])
queryClient.invalidateQueries({
queryKey: consoleQuery.rag.pipelines.datasourcePlugins.get.key(),
})
}, [queryClient, invalidateDataSourceListAuth, invalidateInstalledPluginList])
const toolbar = (
<div

View File

@@ -15,7 +15,6 @@ const mocks = vi.hoisted(() => ({
defaultModel: vi.fn(),
toolProviders: vi.fn(),
builtInTools: vi.fn(),
dataSources: vi.fn(),
dataSourceAuth: vi.fn(),
triggers: vi.fn(),
recommended: vi.fn(),
@@ -34,9 +33,6 @@ vi.mock('@/service/use-tools', () => ({
useInvalidateAllBuiltInTools: () => mocks.builtInTools,
useInvalidateRAGRecommendedPlugins: () => mocks.recommended,
}))
vi.mock('@/service/use-pipeline', () => ({
useInvalidDataSourceList: () => mocks.dataSources,
}))
vi.mock('@/service/use-datasource', () => ({
useInvalidDataSourceListAuth: () => mocks.dataSourceAuth,
}))
@@ -58,6 +54,7 @@ const providerKeys = [
current.modelProviders.summary.get.key(),
commonQueryKeys.modelProviderDetails,
]
const datasourceKey = consoleQuery.rag.pipelines.datasourcePlugins.get.queryKey()
const unrelatedKey = consoleQuery.account.profile.get.key()
const modelTypes = ['llm', 'text-embedding', 'rerank', 'speech2text', 'tts']
@@ -74,6 +71,7 @@ describe('plugin installation refresh', () => {
})
for (const key of [...strategyKeys, ...providerKeys, unrelatedKey])
client.setQueryData(key, { cached: true })
client.setQueryData(datasourceKey, [])
for (const model_type of modelTypes) {
client.setQueryData(
current.models.modelTypes.byModelType.get.queryKey({ input: { params: { model_type } } }),
@@ -107,13 +105,16 @@ describe('plugin installation refresh', () => {
expect(mocks.installed).toHaveBeenCalledWith(category)
for (const key of strategyKeys) expect(client.getQueryState(key)?.isInvalidated).toBe(false)
expect(client.getQueryState(datasourceKey)?.isInvalidated).toBe(
category === PluginCategoryEnum.datasource,
)
if (category === PluginCategoryEnum.tool) {
expect(mocks.toolProviders).toHaveBeenCalledOnce()
expect(mocks.builtInTools).toHaveBeenCalledOnce()
expect(mocks.recommended).toHaveBeenCalledWith('tool')
}
if (category === PluginCategoryEnum.datasource) {
expect(mocks.dataSources).toHaveBeenCalledOnce()
expect(client.getQueryState(datasourceKey)?.isInvalidated).toBe(true)
expect(mocks.dataSourceAuth).toHaveBeenCalledOnce()
}
if (category === PluginCategoryEnum.trigger) expect(mocks.triggers).toHaveBeenCalledOnce()
@@ -135,11 +136,12 @@ describe('plugin installation refresh', () => {
for (const key of providerKeys) expect(client.getQueryState(key)?.isInvalidated).toBe(true)
for (const key of strategyKeys) expect(client.getQueryState(key)?.isInvalidated).toBe(all)
expect(client.getQueryState(unrelatedKey)?.isInvalidated).toBe(false)
expect(client.getQueryState(datasourceKey)?.isInvalidated).toBe(all)
if (all) {
expect(mocks.installed).toHaveBeenCalledWith()
expect(mocks.toolProviders).toHaveBeenCalledOnce()
expect(mocks.builtInTools).toHaveBeenCalledOnce()
expect(mocks.dataSources).toHaveBeenCalledOnce()
expect(client.getQueryState(datasourceKey)?.isInvalidated).toBe(true)
expect(mocks.dataSourceAuth).toHaveBeenCalledOnce()
expect(mocks.triggers).toHaveBeenCalledOnce()
expect(mocks.recommended).toHaveBeenCalledWith('tool')
@@ -158,7 +160,7 @@ describe('plugin installation refresh', () => {
for (const key of [...strategyKeys, ...providerKeys])
expect(client.getQueryState(key)?.isInvalidated).toBe(false)
expect(mocks.toolProviders).not.toHaveBeenCalled()
expect(mocks.dataSources).not.toHaveBeenCalled()
expect(client.getQueryState(datasourceKey)?.isInvalidated).toBe(false)
expect(mocks.triggers).not.toHaveBeenCalled()
expect(mocks.request).not.toHaveBeenCalled()
},

View File

@@ -5,7 +5,6 @@ import { useInvalidateDefaultModel } from '@/app/components/header/account-setti
import { consoleQuery } from '@/service/console'
import { commonQueryKeys } from '@/service/use-common'
import { useInvalidDataSourceListAuth } from '@/service/use-datasource'
import { useInvalidDataSourceList } from '@/service/use-pipeline'
import {
useInvalidateCheckInstalled,
useInvalidateInstalledPluginList,
@@ -63,7 +62,6 @@ const useRefreshPluginList = () => {
const invalidateAllToolProviders = useInvalidateAllToolProviders()
const invalidateAllBuiltInTools = useInvalidateAllBuiltInTools()
const invalidateAllDataSources = useInvalidDataSourceList()
const invalidateDataSourceListAuth = useInvalidDataSourceListAuth()
@@ -95,7 +93,9 @@ const useRefreshPluginList = () => {
(manifest && PluginCategoryEnum.datasource.includes(manifest.category)) ||
refreshAllType
) {
invalidateAllDataSources()
queryClient.invalidateQueries({
queryKey: consoleQuery.rag.pipelines.datasourcePlugins.get.key(),
})
invalidateDataSourceListAuth()
}

View File

@@ -0,0 +1,52 @@
import type { PropsWithChildren } from 'react'
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
import { render, screen, waitFor } from '@testing-library/react'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { consoleQuery } from '@/service/console'
import ActionList from '../datasource-action-list'
const request = vi.hoisted(() => vi.fn<(url: string) => Promise<Response>>())
vi.mock('@/service/base', () => ({ request }))
vi.mock('react-i18next', async () => {
const { createReactI18nextMock } = await import('@/test/i18n-mock')
return createReactI18nextMock({ 'plugin.detailPanel.actionNum': '{{num}} {{action}}' })
})
describe('datasource action list', () => {
let client: QueryClient
const wrapper = ({ children }: PropsWithChildren) => (
<QueryClientProvider client={client}>{children}</QueryClientProvider>
)
beforeEach(() => {
vi.clearAllMocks()
client = new QueryClient({ defaultOptions: { queries: { retry: false } } })
})
afterEach(() => client.clear())
it('retains the zero-action heading for a matching installed provider', async () => {
const provider = createDatasourceProvider()
request.mockResolvedValue(Response.json([provider]))
render(<ActionList detail={{ plugin_id: provider.plugin_id }} />, { wrapper })
expect(await screen.findByText('0 action')).toBeInTheDocument()
expect(screen.queryByRole('button')).not.toBeInTheDocument()
expect(
client.getQueryData(consoleQuery.rag.pipelines.datasourcePlugins.get.queryKey()),
).toEqual([provider])
})
it('does not show the heading for a different installed plugin', async () => {
request.mockResolvedValue(Response.json([createDatasourceProvider()]))
const { container } = render(<ActionList detail={{ plugin_id: 'missing/plugin' }} />, {
wrapper,
})
await waitFor(() =>
expect(
client.getQueryState(consoleQuery.rag.pipelines.datasourcePlugins.get.queryKey())?.status,
).toBe('success'),
)
expect(container).toBeEmptyDOMElement()
})
})

View File

@@ -1,3 +1,4 @@
import type { DatasourceProviderEntity } from '@dify/contracts/api/console/workspaces/types.gen'
import type { PluginDeclaration, PluginDetail } from '@/app/components/plugins/types'
import { fireEvent, render, screen, waitFor } from '@testing-library/react'
import { beforeEach, describe, expect, it, vi } from 'vite-plus/test'
@@ -130,7 +131,6 @@ describe('PluginDetailPanel', () => {
['agent strategy', { agent_strategy: {} }, 'Agent strategies'],
['endpoint', { endpoint: {} }, 'Endpoints'],
['model', { model: {} }, 'Models'],
['data source', { datasource: {} }, 'Data sources'],
] as const)('renders the %s section declared by the plugin', (_, declaration, section) => {
render(
<PluginDetailPanel
@@ -144,6 +144,38 @@ describe('PluginDetailPanel', () => {
expect(screen.getByText('Readme')).toBeInTheDocument()
})
it('renders the datasource section only for a present generated declaration', () => {
const datasource = {
provider_type: 'local_file',
identity: {
author: 'Dify',
name: 'file',
icon: 'file.svg',
label: { en_US: 'Local File' },
description: { en_US: 'Read local files' },
tags: [],
},
oauth_schema: null,
} satisfies DatasourceProviderEntity
const { rerender } = render(
<PluginDetailPanel
detail={createPluginDetail({ datasource })}
onUpdate={onUpdate}
onHide={onHide}
/>,
)
expect(screen.getByText('Data sources')).toBeInTheDocument()
rerender(
<PluginDetailPanel
detail={createPluginDetail({ datasource: null })}
onUpdate={onUpdate}
onHide={onHide}
/>,
)
expect(screen.queryByText('Data sources')).not.toBeInTheDocument()
})
it('renders subscription and event sections for trigger plugins', () => {
render(
<PluginDetailPanel

View File

@@ -1,46 +1,21 @@
// import { Button } from '@langgenius/dify-ui/button'
// import { StatusDot } from '@langgenius/dify-ui/status-dot'
// import ToolItem from '@/app/components/tools/provider/tool-item'
// import ConfigCredential from '@/app/components/tools/setting/build-in/config-credentials'
import type { PluginDetail } from '@/app/components/plugins/types'
import * as React from 'react'
import { useMemo } from 'react'
import { useQuery } from '@tanstack/react-query'
import { useTranslation } from 'react-i18next'
import { transformDataSourceToTool } from '@/app/components/workflow/block-selector/utils'
import { useDataSourceList } from '@/service/use-pipeline'
import { consoleQuery } from '@/service/console'
type Props = Readonly<{
detail: PluginDetail
detail: Pick<PluginDetail, 'plugin_id'>
}>
const ActionList = ({ detail }: Props) => {
const { t } = useTranslation()
// const providerBriefInfo = detail.declaration.datasource?.identity
// const providerKey = `${detail.plugin_id}/${providerBriefInfo?.name}`
const { data: dataSourceList } = useDataSourceList(true)
const provider = useMemo(() => {
const result = dataSourceList?.find((collection) => collection.plugin_id === detail.plugin_id)
const { data: hasProvider } = useQuery(
consoleQuery.rag.pipelines.datasourcePlugins.get.queryOptions({
select: (providers) => providers.some((provider) => provider.plugin_id === detail.plugin_id),
}),
)
if (result) return transformDataSourceToTool(result)
}, [detail.plugin_id, dataSourceList])
const data: any = []
// const { data } = useBuiltinTools(providerKey)
// const [showSettingAuth, setShowSettingAuth] = useState(false)
// const handleCredentialSettingUpdate = () => {
// setShowSettingAuth(false)
// }
// const { mutate: updatePermission, isPending } = useUpdateProviderCredentials({
// onSuccess: handleCredentialSettingUpdate,
// })
// const { mutate: removePermission } = useRemoveProviderCredentials({
// onSuccess: handleCredentialSettingUpdate,
// })
if (!data || !provider) return null
if (!hasProvider) return null
return (
<div className="px-4 pt-2 pb-4">
@@ -48,54 +23,11 @@ const ActionList = ({ detail }: Props) => {
<div className="mb-1 flex h-6 items-center justify-between system-sm-semibold-uppercase text-text-secondary">
{t(($) => $['detailPanel.actionNum'], {
ns: 'plugin',
num: data.length,
action: data.length > 1 ? 'actions' : 'action',
num: 0,
action: 'action',
})}
{/* {provider.is_team_authorization && provider.allow_delete && (
<Button
variant='secondary'
size='small'
onClick={() => setShowSettingAuth(true)}
disabled={!isCurrentWorkspaceManager}
>
<StatusDot className='mr-2' status={'success'} />
{t('tools.auth.authorized')}
</Button>
)} */}
</div>
{/* {!provider.is_team_authorization && provider.allow_delete && (
<Button
variant='primary'
className='w-full'
onClick={() => setShowSettingAuth(true)}
disabled={!isCurrentWorkspaceManager}
>{t('workflow.nodes.tool.authorize')}</Button>
)} */}
</div>
{/* <div className='flex flex-col gap-2'>
{data.map(tool => (
<ToolItem
key={`${detail.plugin_id}${tool.name}`}
disabled={false}
collection={provider}
tool={tool}
isBuiltIn={true}
isModel={false}
/>
))}
</div>
{showSettingAuth && (
<ConfigCredential
collection={provider}
onCancel={() => setShowSettingAuth(false)}
onSaved={async value => updatePermission({
providerName: provider.name,
credentials: value,
})}
onRemove={async () => removePermission(provider.name)}
isSaving={isPending}
/>
)} */}
</div>
)
}

View File

@@ -0,0 +1,60 @@
import { screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { beforeEach, describe, expect, it, vi } from 'vite-plus/test'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { renderWithConsoleQuery as render } from '@/test/console/query-data'
import { ReadmeEntrance } from '../entrance'
import ReadmePanel from '../index'
import { useReadmePanelStore } from '../store'
const { get } = vi.hoisted(() => ({ get: vi.fn() }))
vi.mock('@/service/base', () => ({ get }))
beforeEach(() => {
vi.clearAllMocks()
useReadmePanelStore.setState({ currentPanel: undefined })
get.mockResolvedValue({ readme: '# Datasource documentation' })
})
describe('datasource Readme', () => {
it.each(['drawer', 'dialog'] as const)(
'opens a raw catalog provider in a %s without installed-plugin metadata',
async (presentation) => {
const provider = createDatasourceProvider()
provider.declaration.identity.label = { en_US: 'Datasource display name', zh_Hans: null }
const user = userEvent.setup()
render(
<>
<ReadmeEntrance pluginDetail={provider} presentation={presentation} />
<ReadmePanel />
</>,
)
const trigger = screen.getByRole('button', { name: 'plugin.readmeInfo.needHelpCheckReadme' })
await user.click(trigger)
await vi.dynamicImportSettled()
expect(
await screen.findByRole('heading', { name: 'Datasource documentation' }),
).toBeInTheDocument()
expect(screen.getByText('Datasource display name')).toBeInTheDocument()
expect(screen.getByText('Load files')).toBeInTheDocument()
expect(document.querySelector('img[src="/datasource.svg"]')).toBeInTheDocument()
expect(useReadmePanelStore.getState().currentPanel?.detail).toBe(provider)
expect(get).toHaveBeenCalledWith(
'/workspaces/current/plugin/readme',
{
params: {
plugin_unique_identifier: provider.plugin_unique_identifier,
language: 'en_US',
},
},
{ silent: true },
)
expect(
screen.queryByRole('button', { name: 'plugin.detailPanel.operation.update' }),
).not.toBeInTheDocument()
await user.click(screen.getByRole('button', { name: 'common.operation.close' }))
await waitFor(() => expect(useReadmePanelStore.getState().currentPanel).toBeUndefined())
await waitFor(() => expect(trigger).toHaveFocus())
},
)
})

View File

@@ -1,16 +1,22 @@
'use client'
import type { ReactNode } from 'react'
import type { PluginDetail } from '../types'
import type { ReadmePanelState } from './store'
import { useTranslation } from 'react-i18next'
import { LoadingPlaceholder } from '@/app/components/base/loading-placeholder'
import { Markdown } from '@/app/components/base/markdown'
import { useLanguage } from '@/app/components/header/account-setting/model-provider-page/hooks'
import { resolveDatasourceIcon } from '@/app/components/rag-pipeline/utils/datasource-icon'
import { useGetLanguage } from '@/context/i18n'
import { usePluginReadme } from '@/service/use-plugins'
import Icon from '../card/base/card-icon'
import Description from '../card/base/description'
import OrgInfo from '../card/base/org-info'
import Title from '../card/base/title'
import DetailHeader from '../plugin-detail-panel/detail-header'
type ReadmePanelContentProps = {
detail: PluginDetail
detail: ReadmePanelState['detail']
title: ReactNode
closeButton: ReactNode
}
@@ -18,6 +24,7 @@ type ReadmePanelContentProps = {
export function ReadmePanelContent({ detail, title, closeButton }: ReadmePanelContentProps) {
const { t } = useTranslation()
const language = useLanguage()
const locale = useGetLanguage()
const pluginUniqueIdentifier = detail.plugin_unique_identifier || ''
const {
@@ -70,7 +77,41 @@ export function ReadmePanelContent({ detail, title, closeButton }: ReadmePanelCo
</div>
{closeButton}
</div>
<DetailHeader detail={detail} isReadmeView={true} />
{'id' in detail ? (
<DetailHeader detail={detail} isReadmeView={true} />
) : (
<div>
<div className="flex">
<div className="overflow-hidden rounded-xl border border-components-panel-border-subtle bg-components-panel-bg">
<Icon src={resolveDatasourceIcon(detail.declaration.identity.icon)} />
</div>
<div className="ml-3 min-w-0 grow">
<Title
title={
detail.declaration.identity.label[locale] ??
detail.declaration.identity.label.en_US
}
/>
<OrgInfo
orgName={detail.declaration.identity.author}
packageName={
detail.declaration.identity.name.split('/').pop() ??
detail.declaration.identity.name
}
packageNameClassName="w-auto"
/>
</div>
</div>
<Description
className="mt-2"
text={
detail.declaration.identity.description[locale] ??
detail.declaration.identity.description.en_US
}
descriptionLineRows={2}
/>
</div>
)}
</div>
<div className="min-h-0 flex-1 overflow-y-auto overscroll-contain px-4 py-3">

View File

@@ -1,12 +1,12 @@
'use client'
import type { PluginDetail } from '../types'
import type { ReadmePanelState } from './store'
import { Dialog, DialogClose, DialogContent, DialogTitle } from '@langgenius/dify-ui/dialog'
import { IconButton } from '@langgenius/dify-ui/icon-button'
import { useTranslation } from 'react-i18next'
import { ReadmePanelContent } from './content'
type ReadmeDialogProps = {
detail: PluginDetail
detail: ReadmePanelState['detail']
open: boolean
onOpenChange: (open: boolean) => void
triggerId?: string

View File

@@ -1,6 +1,6 @@
'use client'
import type { PluginDetail } from '../types'
import type { ReadmePanelState } from './store'
import {
Drawer,
DrawerBackdrop,
@@ -15,7 +15,7 @@ import { useTranslation } from 'react-i18next'
import { ReadmePanelContent } from './content'
type ReadmeDrawerProps = {
detail: PluginDetail
detail: ReadmePanelState['detail']
open: boolean
onOpenChange: (open: boolean) => void
triggerId?: string

View File

@@ -1,5 +1,4 @@
import type { PluginDetail } from '../types'
import type { ReadmePanelPresentation } from './store'
import type { ReadmePanelPresentation, ReadmePanelState } from './store'
import { cn } from '@langgenius/dify-ui/cn'
import { useId } from 'react'
import { useTranslation } from 'react-i18next'
@@ -12,7 +11,7 @@ export const ReadmeEntrance = ({
className,
showShortTip = false,
}: {
pluginDetail: PluginDetail
pluginDetail: ReadmePanelState['detail']
presentation?: ReadmePanelPresentation
className?: string
showShortTip?: boolean
@@ -33,7 +32,7 @@ export const ReadmeEntrance = ({
if (
!pluginDetail ||
!pluginDetail?.plugin_unique_identifier ||
BUILTIN_TOOLS_ARRAY.includes(pluginDetail.id)
('id' in pluginDetail && BUILTIN_TOOLS_ARRAY.includes(pluginDetail.id))
)
return null

View File

@@ -1,16 +1,17 @@
import type { RagPipelineDatasourceProviderResponse } from '@dify/contracts/api/console/rag/types.gen'
import type { PluginDetail } from '@/app/components/plugins/types'
import { create } from 'zustand'
export type ReadmePanelPresentation = 'drawer' | 'dialog'
type ReadmePanelState = {
detail: PluginDetail
export type ReadmePanelState = {
detail: PluginDetail | RagPipelineDatasourceProviderResponse
presentation: ReadmePanelPresentation
triggerId?: string
}
type OpenReadmePanelPayload = {
detail: PluginDetail
detail: ReadmePanelState['detail']
presentation?: ReadmePanelPresentation
triggerId?: string
}

View File

@@ -1,5 +1,6 @@
import type {
AgentStrategyProviderEntity,
DatasourceProviderEntity,
EndpointProviderDeclarationResponse,
} from '@dify/contracts/api/console/workspaces/types.gen'
import type { FormTypeEnum } from '../base/form/types'
@@ -64,7 +65,7 @@ export type PluginDeclaration = {
verified: boolean
endpoint?: EndpointProviderDeclarationResponse | null
tool?: PluginToolDeclaration
datasource?: PluginToolDeclaration
datasource?: DatasourceProviderEntity | null
model: any
tags: string[]
agent_strategy?: AgentStrategyProviderEntity | null

View File

@@ -0,0 +1,39 @@
import type { RagPipelineDatasourceProviderResponse } from '@dify/contracts/api/console/rag/types.gen'
import { zRagPipelineDatasourceProviderResponse } from '@dify/contracts/api/console/rag/zod.gen'
export function createDatasourceProvider(
overrides: Partial<RagPipelineDatasourceProviderResponse> = {},
) {
return zRagPipelineDatasourceProviderResponse.parse({
plugin_id: 'langgenius/file',
plugin_unique_identifier: 'langgenius/file:1.0.0',
provider: 'file',
is_authorized: true,
declaration: {
provider_type: 'local_file',
identity: {
author: 'Dify',
name: 'langgenius/file/file',
label: { en_US: 'File Source', zh_Hans: '文件源' },
description: { en_US: 'Load files', zh_Hans: '加载文件' },
icon: '/datasource.svg',
tags: [],
},
credentials_schema: [],
datasources: [
{
identity: {
author: 'Dify',
name: 'local-file',
provider: 'langgenius/file/file',
label: { en_US: 'Local File', zh_Hans: '本地文件' },
},
description: { en_US: 'Load local files', zh_Hans: '加载本地文件' },
parameters: [],
output_schema: null,
},
],
},
...overrides,
})
}

View File

@@ -0,0 +1,92 @@
# Datasource catalog
`GET /rag/pipelines/datasource-plugins` owns the workspace's available datasource
providers. The controller exports `RagPipelineDatasourceListResponse` from the
existing plugin datasource domain entities. Generated Console types are the wire
contract; consumers must not recreate a `DataSourceItem` or cast providers and
parameters to tool DTOs.
## Data ownership
The plugin manager inserts local-file support, resolves output-schema references,
and normalizes declaration identities. The top-level `provider` is the original
provider name; `declaration.identity.name` and each datasource's
`identity.provider` contain the plugin/provider path. Saved workflow nodes retain
the original provider name because execution joins it with `plugin_id`.
The pipeline service derives `is_authorized` from workspace credentials. A provider
without credential or OAuth requirements is available without authorization.
Credential lookup failures leave it unauthorized. This field describes workspace
availability; it does not grant a user permission to manage credentials.
The Query cache and workflow store retain the generated response without mutation.
Icon URLs are projected at display boundaries: root-relative URLs receive
`basePath` once; absolute and protocol-relative URLs remain intact.
Installed-plugin manifests use the generated `DatasourceProviderEntity` for
their datasource declaration. This declaration is not the catalog response: it
has no expanded action list or workspace authorization state. The unknown
manifest input is validated once with its generated Zod schema; absent and null
declarations remain absent and null. Plugin details only use its presence to
admit the datasource section.
## Query and refresh ownership
All consumers use `consoleQuery.rag.pipelines.datasourcePlugins.get.queryOptions()`.
The operation policy uses `staleTime: 0` and `retry: false`, preserving immediate
availability checks without retrying an unavailable plugin catalog.
| Consumer | Admission and local responsibility |
| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ |
| Pipeline configuration | Requires a pipeline ID. Its observer synchronizes cached data and subsequent refreshes into the workflow store, including an empty result. |
| Document source options | Reads the mounted source's icon without changing cached provider data. |
| Deployment precheck | Fetches only when an unsupported node is a datasource. |
| Plugin detail | Checks whether the plugin has a catalog provider. Retains the existing zero-action heading; it does not enable a new action browser. |
Plugin installation, datasource node installation, credential reconciliation, and
the datasource settings page refresh the generated catalog key directly. Each
owner calls `QueryClient.invalidateQueries` inline. This read-only catalog has no
mutation endpoint and adds no mutation policy. Credential writes and OAuth
lifecycles belong to the authorization module; they must also reconcile catalog
availability when credentials change.
## Type and presentation boundaries
`WorkflowPluginCatalogs` names each catalog explicitly: tool collections keep
their tool type, while `dataSourceList` holds generated datasource providers.
Common node fields explicitly select `provider_type`, `provider_name`, `plugin_id`,
and `plugin_unique_identifier` from plugin defaults;
tool parameters and output schemas belong to their specific node payloads.
The pipeline slice declares its complete output separately from the partially
injected workflow state, so context injection needs no type assertion.
The datasource selector consumes provider declarations directly and creates
`DataSourceDefaultValue` without an intermediate tool value. It preserves provider
search, localized grouping, marketplace discovery, and local-file defaults. A
saved node can still resolve a provider without newer plugin version metadata.
Datasource parameters have their own generated parameter union. Their form
projection reads `description`, retains `false` and `0`, and preserves numeric
bounds. The shared parameter form receives an explicit static-schema capability;
it does not send datasource identities to tool or trigger dynamic-option APIs.
Credential-management schema types are not widened to describe node parameter
values.
Output schemas may contain arbitrary JSON. The variable and read-only schema
presentations narrow supported properties, types, descriptions, and enums at the
point of use. Unknown or boolean schema nodes do not justify changing backend
JSON-schema semantics or asserting an LLM editor schema. LLM schema editing keeps
its own contract.
## Provider identity and Readme
Saved nodes resolve an exact installed identifier first, then the same plugin ID
when its installed version has changed. Provider-name fallback is only for saved
nodes without either plugin identity; another plugin with the same provider name
must not supply authorization, parameters, outputs, or installation status.
The workflow Readme entry passes the raw catalog provider through the Readme
session. Its header reads `declaration.identity` and has no installed-plugin
management actions. Full installed-plugin details retain their own header.
Catalog providers are not cast to `PluginDetail`, and no synthetic installation
fields are added to make that presentation work.

View File

@@ -1,275 +1,155 @@
import { renderHook } from '@testing-library/react'
import type { PropsWithChildren } from 'react'
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
import { act, renderHook, waitFor } from '@testing-library/react'
import { createElement } from 'react'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test'
import { WorkflowContext } from '@/app/components/workflow/context'
import { createWorkflowStore } from '@/app/components/workflow/store'
import { consoleQuery } from '@/service/console'
import { createDatasourceProvider } from '../../__tests__/datasource-fixtures'
import { createRagPipelineSliceSlice } from '../../store'
import { usePipelineConfig } from '../use-pipeline-config'
const mockUseStore = vi.fn()
const mockWorkflowStoreGetState = vi.fn()
vi.mock('@/app/components/workflow/store', () => ({
useStore: (selector: (state: Record<string, unknown>) => unknown) => mockUseStore(selector),
useWorkflowStore: () => ({
getState: mockWorkflowStoreGetState,
}),
const mocks = vi.hoisted(() => ({
request: vi.fn<(url: string) => Promise<Response>>(),
workflowConfig: vi.fn<(url: string, callback: (data: unknown) => void) => void>(),
}))
vi.mock('@/service/base', () => ({ request: mocks.request }))
vi.mock('@/service/use-workflow', () => ({ useWorkflowConfig: mocks.workflowConfig }))
const mockUseWorkflowConfig = vi.fn()
vi.mock('@/service/use-workflow', () => ({
useWorkflowConfig: (url: string, callback: (data: unknown) => void) =>
mockUseWorkflowConfig(url, callback),
}))
const mockUseDataSourceList = vi.fn()
vi.mock('@/service/use-pipeline', () => ({
useDataSourceList: (enabled: boolean, callback: (data: unknown) => void) =>
mockUseDataSourceList(enabled, callback),
}))
vi.mock('@/utils/var', () => ({
basePath: '/base',
}))
const catalog = consoleQuery.rag.pipelines.datasourcePlugins.get
describe('usePipelineConfig', () => {
const mockSetNodesDefaultConfigs = vi.fn()
const mockSetPublishedAt = vi.fn()
const mockSetDataSourceList = vi.fn()
const mockSetFileUploadConfig = vi.fn()
let client: QueryClient
let store: ReturnType<typeof createWorkflowStore>
const wrapper = ({ children }: PropsWithChildren) =>
createElement(
QueryClientProvider,
{ client },
createElement(WorkflowContext.Provider, { value: store }, children),
)
beforeEach(() => {
vi.clearAllMocks()
client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: Infinity } } })
store = createWorkflowStore({ injectWorkflowStoreSliceFn: createRagPipelineSliceSlice })
store.setState({ pipelineId: 'pipeline-1' })
mocks.request.mockResolvedValue(Response.json([]))
})
afterEach(() => client.clear())
mockUseStore.mockImplementation((selector: (state: Record<string, unknown>) => unknown) => {
const state = { pipelineId: 'test-pipeline-id' }
return selector(state)
})
it('hydrates the store from cached raw data before refreshing it', async () => {
const cached = [createDatasourceProvider()]
client.setQueryData(catalog.queryKey(), cached)
let resolveRequest!: (response: Response) => void
mocks.request.mockReturnValue(
new Promise((resolve) => {
resolveRequest = resolve
}),
)
mockWorkflowStoreGetState.mockReturnValue({
setNodesDefaultConfigs: mockSetNodesDefaultConfigs,
setPublishedAt: mockSetPublishedAt,
setDataSourceList: mockSetDataSourceList,
setFileUploadConfig: mockSetFileUploadConfig,
})
renderHook(usePipelineConfig, { wrapper })
expect(store.getState().dataSourceList).toEqual(cached)
expect(store.getState().dataSourceList?.[0]?.declaration.identity.icon).toBe('/datasource.svg')
const refreshed = [createDatasourceProvider({ is_authorized: false })]
await act(async () => resolveRequest(Response.json(refreshed)))
await waitFor(() => expect(store.getState().dataSourceList).toEqual(refreshed))
expect(mocks.request).toHaveBeenCalledOnce()
expect(mocks.request.mock.calls[0]?.[0]).toContain('/rag/pipelines/datasource-plugins')
expect(client.getQueryData(catalog.queryKey())).toEqual(refreshed)
expect(cached[0]?.declaration.identity.icon).toBe('/datasource.svg')
})
afterEach(() => {
vi.clearAllMocks()
it('reconciles an empty catalog on background invalidation', async () => {
const providers = [createDatasourceProvider()]
mocks.request.mockResolvedValueOnce(Response.json(providers))
renderHook(usePipelineConfig, { wrapper })
await waitFor(() => expect(store.getState().dataSourceList).toEqual(providers))
await act(async () => client.invalidateQueries({ queryKey: catalog.key() }))
await waitFor(() => expect(store.getState().dataSourceList).toEqual([]))
expect(mocks.request).toHaveBeenCalledTimes(2)
})
describe('hook initialization', () => {
it('should call useWorkflowConfig with correct URL for nodes default configs', () => {
renderHook(() => usePipelineConfig())
it('does not request or hydrate datasource state before a pipeline is available', () => {
store.setState({ pipelineId: '' })
client.setQueryData(catalog.queryKey(), [createDatasourceProvider()])
renderHook(usePipelineConfig, { wrapper })
expect(mockUseWorkflowConfig).toHaveBeenCalledWith(
'/rag/pipelines/test-pipeline-id/workflows/default-workflow-block-configs',
expect.any(Function),
)
})
it('should call useWorkflowConfig with correct URL for published workflow', () => {
renderHook(() => usePipelineConfig())
expect(mockUseWorkflowConfig).toHaveBeenCalledWith(
'/rag/pipelines/test-pipeline-id/workflows/publish',
expect.any(Function),
)
})
it('should call useWorkflowConfig with correct URL for file upload config', () => {
renderHook(() => usePipelineConfig())
expect(mockUseWorkflowConfig).toHaveBeenCalledWith('/files/upload', expect.any(Function))
})
it('should call useDataSourceList when pipelineId exists', () => {
renderHook(() => usePipelineConfig())
expect(mockUseDataSourceList).toHaveBeenCalledWith(true, expect.any(Function))
})
it('should call useDataSourceList with false when pipelineId is missing', () => {
mockUseStore.mockImplementation((selector: (state: Record<string, unknown>) => unknown) => {
const state = { pipelineId: undefined }
return selector(state)
})
renderHook(() => usePipelineConfig())
expect(mockUseDataSourceList).toHaveBeenCalledWith(false, expect.any(Function))
})
it('should use empty URL when pipelineId is missing for nodes configs', () => {
mockUseStore.mockImplementation((selector: (state: Record<string, unknown>) => unknown) => {
const state = { pipelineId: undefined }
return selector(state)
})
renderHook(() => usePipelineConfig())
expect(mockUseWorkflowConfig).toHaveBeenCalledWith('', expect.any(Function))
})
expect(mocks.request).not.toHaveBeenCalled()
expect(store.getState().dataSourceList).toEqual([])
expect(mocks.workflowConfig).toHaveBeenCalledWith('', expect.any(Function))
})
describe('handleUpdateNodesDefaultConfigs', () => {
it('should handle array format configs', () => {
let capturedCallback: ((data: unknown) => void) | undefined
mockUseWorkflowConfig.mockImplementation((url: string, callback: (data: unknown) => void) => {
if (url.includes('default-workflow-block-configs')) {
capturedCallback = callback
}
})
it('fetches and synchronizes when the pipeline becomes available', async () => {
store.setState({ pipelineId: '' })
const providers = [createDatasourceProvider()]
mocks.request.mockResolvedValue(Response.json(providers))
renderHook(usePipelineConfig, { wrapper })
renderHook(() => usePipelineConfig())
act(() => store.setState({ pipelineId: 'pipeline-2' }))
const arrayConfigs = [
{ type: 'llm', config: { model: 'gpt-4' } },
{ type: 'code', config: { language: 'python' } },
]
capturedCallback?.(arrayConfigs)
expect(mockSetNodesDefaultConfigs).toHaveBeenCalledWith({
llm: { model: 'gpt-4' },
code: { language: 'python' },
})
})
it('should handle object format configs', () => {
let capturedCallback: ((data: unknown) => void) | undefined
mockUseWorkflowConfig.mockImplementation((url: string, callback: (data: unknown) => void) => {
if (url.includes('default-workflow-block-configs')) {
capturedCallback = callback
}
})
renderHook(() => usePipelineConfig())
const objectConfigs = {
llm: { model: 'gpt-4' },
code: { language: 'python' },
}
capturedCallback?.(objectConfigs)
expect(mockSetNodesDefaultConfigs).toHaveBeenCalledWith(objectConfigs)
})
await waitFor(() => expect(store.getState().dataSourceList).toEqual(providers))
})
describe('handleUpdatePublishedAt', () => {
it('should set published at from workflow response', () => {
let capturedCallback: ((data: unknown) => void) | undefined
mockUseWorkflowConfig.mockImplementation((url: string, callback: (data: unknown) => void) => {
if (url.includes('/publish')) {
capturedCallback = callback
}
})
it('keeps cached data when refreshing fails without retrying the availability request', async () => {
const providers = [createDatasourceProvider()]
client.setQueryData(catalog.queryKey(), providers)
mocks.request.mockRejectedValue(new Error('Datasource catalog unavailable'))
renderHook(usePipelineConfig, { wrapper })
renderHook(() => usePipelineConfig())
capturedCallback?.({ created_at: '2024-01-01T00:00:00Z' })
expect(mockSetPublishedAt).toHaveBeenCalledWith('2024-01-01T00:00:00Z')
})
it('should reset published at when workflow response is empty', () => {
let capturedCallback: ((data: unknown) => void) | undefined
mockUseWorkflowConfig.mockImplementation((url: string, callback: (data: unknown) => void) => {
if (url.includes('/publish')) {
capturedCallback = callback
}
})
renderHook(() => usePipelineConfig())
capturedCallback?.(undefined)
expect(mockSetPublishedAt).toHaveBeenCalledWith(0)
})
await waitFor(() => expect(client.getQueryState(catalog.queryKey())?.status).toBe('error'))
expect(client.getQueryState(catalog.queryKey())?.fetchFailureCount).toBe(1)
expect(mocks.request).toHaveBeenCalledOnce()
expect(store.getState().dataSourceList).toEqual(providers)
})
describe('handleUpdateDataSourceList', () => {
it('should set data source list', () => {
let capturedCallback: ((data: unknown) => void) | undefined
mockUseDataSourceList.mockImplementation(
(_enabled: boolean, callback: (data: unknown) => void) => {
capturedCallback = callback
},
)
it('shares the raw catalog with other observers without transforming provider data', async () => {
const providers = [createDatasourceProvider()]
mocks.request.mockResolvedValue(Response.json(providers))
renderHook(usePipelineConfig, { wrapper })
await waitFor(() => expect(store.getState().dataSourceList).toEqual(providers))
renderHook(() => usePipelineConfig())
const dataSourceList = [{ declaration: { identity: { icon: '/icon.png' } } }]
capturedCallback?.(dataSourceList)
expect(mockSetDataSourceList).toHaveBeenCalled()
})
it('should prepend basePath to icon if not included', () => {
let capturedCallback: ((data: unknown) => void) | undefined
mockUseDataSourceList.mockImplementation(
(_enabled: boolean, callback: (data: unknown) => void) => {
capturedCallback = callback
},
)
renderHook(() => usePipelineConfig())
const dataSourceList = [{ declaration: { identity: { icon: '/icon.png' } } }]
capturedCallback?.(dataSourceList)
expect(dataSourceList[0]!.declaration.identity.icon).toBe('/base/icon.png')
})
it('should not modify icon if it already includes basePath', () => {
let capturedCallback: ((data: unknown) => void) | undefined
mockUseDataSourceList.mockImplementation(
(_enabled: boolean, callback: (data: unknown) => void) => {
capturedCallback = callback
},
)
renderHook(() => usePipelineConfig())
const dataSourceList = [{ declaration: { identity: { icon: '/base/icon.png' } } }]
capturedCallback?.(dataSourceList)
expect(dataSourceList[0]!.declaration.identity.icon).toBe('/base/icon.png')
})
it('should handle non-string icon', () => {
let capturedCallback: ((data: unknown) => void) | undefined
mockUseDataSourceList.mockImplementation(
(_enabled: boolean, callback: (data: unknown) => void) => {
capturedCallback = callback
},
)
renderHook(() => usePipelineConfig())
const dataSourceList = [{ declaration: { identity: { icon: { url: '/icon.png' } } } }]
capturedCallback?.(dataSourceList)
expect(dataSourceList[0]!.declaration.identity.icon).toEqual({ url: '/icon.png' })
})
expect(store.getState().dataSourceList).toBe(client.getQueryData(catalog.queryKey()))
expect(catalog.queryOptions()).toMatchObject({ staleTime: 0, retry: false })
})
describe('handleUpdateWorkflowFileUploadConfig', () => {
it('should set file upload config', () => {
let capturedCallback: ((data: unknown) => void) | undefined
mockUseWorkflowConfig.mockImplementation((url: string, callback: (data: unknown) => void) => {
if (url === '/files/upload') {
capturedCallback = callback
}
})
renderHook(() => usePipelineConfig())
const config = { max_file_size: 10 * 1024 * 1024 }
capturedCallback?.(config)
expect(mockSetFileUploadConfig).toHaveBeenCalledWith(config)
})
it('retains the other pipeline configuration entrypoints', () => {
renderHook(usePipelineConfig, { wrapper })
expect(mocks.workflowConfig).toHaveBeenCalledWith(
'/rag/pipelines/pipeline-1/workflows/default-workflow-block-configs',
expect.any(Function),
)
expect(mocks.workflowConfig).toHaveBeenCalledWith(
'/rag/pipelines/pipeline-1/workflows/publish',
expect.any(Function),
)
expect(mocks.workflowConfig).toHaveBeenCalledWith('/files/upload', expect.any(Function))
})
it.each([[{ type: 'llm', config: { model: 'test' } }], { llm: { model: 'test' } }])(
'retains default node configuration normalization',
(config) => {
renderHook(usePipelineConfig, { wrapper })
const callback = mocks.workflowConfig.mock.calls.find(([url]) =>
url.endsWith('default-workflow-block-configs'),
)?.[1]
act(() => callback?.(config))
expect(store.getState().nodesDefaultConfigs).toEqual({ llm: { model: 'test' } })
},
)
it.each([{ created_at: 123 }, null])(
'retains the publication timestamp fallback',
(published) => {
renderHook(usePipelineConfig, { wrapper })
const callback = mocks.workflowConfig.mock.calls.find(([url]) =>
url.endsWith('/publish'),
)?.[1]
act(() => callback?.(published))
expect(store.getState().publishedAt).toBe((published?.created_at ?? 0) * 1000)
},
)
})

View File

@@ -1,11 +1,10 @@
import type { DataSourceItem } from '@/app/components/workflow/block-selector/types'
import type { FileUploadConfigResponse } from '@/models/common'
import type { FetchWorkflowDraftResponse } from '@/types/workflow'
import { useCallback } from 'react'
import { useQuery } from '@tanstack/react-query'
import { useCallback, useEffect } from 'react'
import { useStore, useWorkflowStore } from '@/app/components/workflow/store'
import { useDataSourceList } from '@/service/use-pipeline'
import { consoleQuery } from '@/service/console'
import { useWorkflowConfig } from '@/service/use-workflow'
import { basePath } from '@/utils/var'
export const usePipelineConfig = () => {
const pipelineId = useStore((s) => s.pipelineId)
@@ -45,19 +44,6 @@ export const usePipelineConfig = () => {
handleUpdatePublishedAt,
)
const handleUpdateDataSourceList = useCallback(
(dataSourceList: DataSourceItem[]) => {
dataSourceList.forEach((item) => {
const icon = item.declaration.identity.icon
if (typeof icon == 'string' && !icon.includes(basePath))
item.declaration.identity.icon = `${basePath}${icon}`
})
const { setDataSourceList } = workflowStore.getState()
setDataSourceList!(dataSourceList)
},
[workflowStore],
)
const handleUpdateWorkflowFileUploadConfig = useCallback(
(config: FileUploadConfigResponse) => {
const { setFileUploadConfig } = workflowStore.getState()
@@ -67,5 +53,13 @@ export const usePipelineConfig = () => {
)
useWorkflowConfig('/files/upload', handleUpdateWorkflowFileUploadConfig)
useDataSourceList(!!pipelineId, handleUpdateDataSourceList)
const { data: dataSourceList } = useQuery(
consoleQuery.rag.pipelines.datasourcePlugins.get.queryOptions({
enabled: !!pipelineId,
}),
)
useEffect(() => {
if (!pipelineId || !dataSourceList) return
workflowStore.getState().setDataSourceList?.(dataSourceList)
}, [dataSourceList, pipelineId, workflowStore])
}

View File

@@ -1,4 +1,3 @@
import type { InjectWorkflowStoreSliceFn } from '@/app/components/workflow/store'
import { useMemo } from 'react'
import { LoadingPlaceholder } from '@/app/components/base/loading-placeholder'
import WorkflowWithDefaultContext from '@/app/components/workflow'
@@ -49,9 +48,7 @@ const RagPipelineWrapper = () => {
if (!pipelineId) return <Conversion />
return (
<WorkflowContextProvider
injectWorkflowStoreSliceFn={createRagPipelineSliceSlice as InjectWorkflowStoreSliceFn}
>
<WorkflowContextProvider injectWorkflowStoreSliceFn={createRagPipelineSliceSlice}>
<RagPipeline />
</WorkflowContextProvider>
)

View File

@@ -1,295 +1,22 @@
import type { InputFieldEditorProps } from '../../components/panel/input-field/editor'
import type { RagPipelineSliceShape } from '../index'
import type { DataSourceItem } from '@/app/components/workflow/block-selector/types'
import type { RAGPipelineVariables } from '@/models/pipeline'
import { describe, expect, it, vi } from 'vite-plus/test'
import { PipelineInputVarType } from '@/models/pipeline'
import { createStore } from 'zustand/vanilla'
import { createDatasourceProvider } from '../../__tests__/datasource-fixtures'
import { createRagPipelineSliceSlice } from '../index'
vi.mock('@/app/components/workflow/block-selector/utils', () => ({
transformDataSourceToTool: (item: DataSourceItem) => ({
...item,
transformed: true,
}),
}))
it('stores the datasource domain unchanged and replaces it on catalog refresh', () => {
const store = createStore<RagPipelineSliceShape>()(createRagPipelineSliceSlice)
const provider = createDatasourceProvider({ is_authorized: false })
type SliceCreatorParams = Parameters<typeof createRagPipelineSliceSlice>
const unusedGet = vi.fn() as unknown as SliceCreatorParams[1]
const unusedApi = vi.fn() as unknown as SliceCreatorParams[2]
store.getState().setDataSourceList([provider])
function createSlice(mockSet = vi.fn()) {
return createRagPipelineSliceSlice(
mockSet as unknown as SliceCreatorParams[0],
unusedGet,
unusedApi,
)
}
expect(store.getState().dataSourceList).toEqual([provider])
expect(store.getState().dataSourceList[0]).not.toHaveProperty('tools')
expect(store.getState().dataSourceList[0]?.declaration.identity.icon).toBe('/datasource.svg')
describe('createRagPipelineSliceSlice', () => {
const mockSet = vi.fn()
describe('initial state', () => {
it('should have empty pipelineId', () => {
const slice = createSlice(mockSet)
expect(slice.pipelineId).toBe('')
})
it('should have empty knowledgeName', () => {
const slice = createSlice(mockSet)
expect(slice.knowledgeName).toBe('')
})
it('should have showInputFieldPanel as false', () => {
const slice = createSlice(mockSet)
expect(slice.showInputFieldPanel).toBe(false)
})
it('should have showInputFieldPreviewPanel as false', () => {
const slice = createSlice(mockSet)
expect(slice.showInputFieldPreviewPanel).toBe(false)
})
it('should have inputFieldEditPanelProps as null', () => {
const slice = createSlice(mockSet)
expect(slice.inputFieldEditPanelProps).toBeNull()
})
it('should have empty nodesDefaultConfigs', () => {
const slice = createSlice(mockSet)
expect(slice.nodesDefaultConfigs).toEqual({})
})
it('should have empty ragPipelineVariables', () => {
const slice = createSlice(mockSet)
expect(slice.ragPipelineVariables).toEqual([])
})
it('should have empty dataSourceList', () => {
const slice = createSlice(mockSet)
expect(slice.dataSourceList).toEqual([])
})
it('should have isPreparingDataSource as false', () => {
const slice = createSlice(mockSet)
expect(slice.isPreparingDataSource).toBe(false)
})
})
describe('setShowInputFieldPanel', () => {
it('should call set with showInputFieldPanel true', () => {
const slice = createSlice(mockSet)
slice.setShowInputFieldPanel(true)
expect(mockSet).toHaveBeenCalledWith(expect.any(Function))
const setterFn = mockSet.mock.calls[0]![0] as () => Partial<RagPipelineSliceShape>
const result = setterFn()
expect(result).toEqual({ showInputFieldPanel: true })
})
it('should call set with showInputFieldPanel false', () => {
mockSet.mockClear()
const slice = createSlice(mockSet)
slice.setShowInputFieldPanel(false)
const setterFn = mockSet.mock.calls[0]![0] as () => Partial<RagPipelineSliceShape>
const result = setterFn()
expect(result).toEqual({ showInputFieldPanel: false })
})
})
describe('setShowInputFieldPreviewPanel', () => {
it('should call set with showInputFieldPreviewPanel true', () => {
mockSet.mockClear()
const slice = createSlice(mockSet)
slice.setShowInputFieldPreviewPanel(true)
const setterFn = mockSet.mock.calls[0]![0] as () => Partial<RagPipelineSliceShape>
const result = setterFn()
expect(result).toEqual({ showInputFieldPreviewPanel: true })
})
it('should call set with showInputFieldPreviewPanel false', () => {
mockSet.mockClear()
const slice = createSlice(mockSet)
slice.setShowInputFieldPreviewPanel(false)
const setterFn = mockSet.mock.calls[0]![0] as () => Partial<RagPipelineSliceShape>
const result = setterFn()
expect(result).toEqual({ showInputFieldPreviewPanel: false })
})
})
describe('setInputFieldEditPanelProps', () => {
it('should call set with inputFieldEditPanelProps object', () => {
mockSet.mockClear()
const slice = createSlice(mockSet)
const props = { onClose: vi.fn(), onSubmit: vi.fn() } as unknown as InputFieldEditorProps
slice.setInputFieldEditPanelProps(props)
const setterFn = mockSet.mock.calls[0]![0] as () => Partial<RagPipelineSliceShape>
const result = setterFn()
expect(result).toEqual({ inputFieldEditPanelProps: props })
})
it('should call set with inputFieldEditPanelProps null', () => {
mockSet.mockClear()
const slice = createSlice(mockSet)
slice.setInputFieldEditPanelProps(null)
const setterFn = mockSet.mock.calls[0]![0] as () => Partial<RagPipelineSliceShape>
const result = setterFn()
expect(result).toEqual({ inputFieldEditPanelProps: null })
})
})
describe('setNodesDefaultConfigs', () => {
it('should call set with nodesDefaultConfigs', () => {
mockSet.mockClear()
const slice = createSlice(mockSet)
const configs: Record<string, unknown> = { node1: { key: 'value' } }
slice.setNodesDefaultConfigs(configs)
const setterFn = mockSet.mock.calls[0]![0] as () => Partial<RagPipelineSliceShape>
const result = setterFn()
expect(result).toEqual({ nodesDefaultConfigs: configs })
})
it('should call set with empty nodesDefaultConfigs', () => {
mockSet.mockClear()
const slice = createSlice(mockSet)
slice.setNodesDefaultConfigs({})
const setterFn = mockSet.mock.calls[0]![0] as () => Partial<RagPipelineSliceShape>
const result = setterFn()
expect(result).toEqual({ nodesDefaultConfigs: {} })
})
})
describe('setRagPipelineVariables', () => {
it('should call set with ragPipelineVariables', () => {
mockSet.mockClear()
const slice = createSlice(mockSet)
const variables: RAGPipelineVariables = [
{
type: PipelineInputVarType.textInput,
variable: 'var1',
label: 'Var 1',
required: true,
belong_to_node_id: 'node-1',
},
]
slice.setRagPipelineVariables(variables)
const setterFn = mockSet.mock.calls[0]![0] as () => Partial<RagPipelineSliceShape>
const result = setterFn()
expect(result).toEqual({ ragPipelineVariables: variables })
})
it('should call set with empty ragPipelineVariables', () => {
mockSet.mockClear()
const slice = createSlice(mockSet)
slice.setRagPipelineVariables([])
const setterFn = mockSet.mock.calls[0]![0] as () => Partial<RagPipelineSliceShape>
const result = setterFn()
expect(result).toEqual({ ragPipelineVariables: [] })
})
})
describe('setDataSourceList', () => {
it('should transform and set dataSourceList', () => {
mockSet.mockClear()
const slice = createSlice(mockSet)
const dataSourceList: DataSourceItem[] = [
{ name: 'source1', key: 'key1' } as unknown as DataSourceItem,
{ name: 'source2', key: 'key2' } as unknown as DataSourceItem,
]
slice.setDataSourceList(dataSourceList)
const setterFn = mockSet.mock.calls[0]![0] as () => Partial<RagPipelineSliceShape>
const result = setterFn()
expect(result.dataSourceList).toHaveLength(2)
expect(result.dataSourceList![0]).toEqual({ name: 'source1', key: 'key1', transformed: true })
expect(result.dataSourceList![1]).toEqual({ name: 'source2', key: 'key2', transformed: true })
})
it('should set empty dataSourceList', () => {
mockSet.mockClear()
const slice = createSlice(mockSet)
slice.setDataSourceList([])
const setterFn = mockSet.mock.calls[0]![0] as () => Partial<RagPipelineSliceShape>
const result = setterFn()
expect(result.dataSourceList).toEqual([])
})
})
describe('setIsPreparingDataSource', () => {
it('should call set with isPreparingDataSource true', () => {
mockSet.mockClear()
const slice = createSlice(mockSet)
slice.setIsPreparingDataSource(true)
const setterFn = mockSet.mock.calls[0]![0] as () => Partial<RagPipelineSliceShape>
const result = setterFn()
expect(result).toEqual({ isPreparingDataSource: true })
})
it('should call set with isPreparingDataSource false', () => {
mockSet.mockClear()
const slice = createSlice(mockSet)
slice.setIsPreparingDataSource(false)
const setterFn = mockSet.mock.calls[0]![0] as () => Partial<RagPipelineSliceShape>
const result = setterFn()
expect(result).toEqual({ isPreparingDataSource: false })
})
})
})
describe('RagPipelineSliceShape type', () => {
it('should define all required properties', () => {
const slice = createSlice()
expect(slice).toHaveProperty('pipelineId')
expect(slice).toHaveProperty('knowledgeName')
expect(slice).toHaveProperty('showInputFieldPanel')
expect(slice).toHaveProperty('setShowInputFieldPanel')
expect(slice).toHaveProperty('showInputFieldPreviewPanel')
expect(slice).toHaveProperty('setShowInputFieldPreviewPanel')
expect(slice).toHaveProperty('inputFieldEditPanelProps')
expect(slice).toHaveProperty('setInputFieldEditPanelProps')
expect(slice).toHaveProperty('nodesDefaultConfigs')
expect(slice).toHaveProperty('setNodesDefaultConfigs')
expect(slice).toHaveProperty('ragPipelineVariables')
expect(slice).toHaveProperty('setRagPipelineVariables')
expect(slice).toHaveProperty('dataSourceList')
expect(slice).toHaveProperty('setDataSourceList')
expect(slice).toHaveProperty('isPreparingDataSource')
expect(slice).toHaveProperty('setIsPreparingDataSource')
})
const authorized = { ...provider, is_authorized: true }
store.getState().setDataSourceList([authorized])
expect(store.getState().dataSourceList).toEqual([authorized])
expect(provider.is_authorized).toBe(false)
store.getState().setDataSourceList([])
expect(store.getState().dataSourceList).toEqual([])
})

View File

@@ -1,10 +1,8 @@
import type { RagPipelineDatasourceProviderResponse } from '@dify/contracts/api/console/rag/types.gen'
import type { StateCreator } from 'zustand'
import type { InputFieldEditorProps } from '../components/panel/input-field/editor'
import type { DataSourceItem } from '@/app/components/workflow/block-selector/types'
import type { ToolWithProvider } from '@/app/components/workflow/types'
import type { IconInfo } from '@/models/datasets'
import type { RAGPipelineVariables } from '@/models/pipeline'
import { transformDataSourceToTool } from '@/app/components/workflow/block-selector/utils'
export type RagPipelineSliceShape = {
pipelineId: string
@@ -20,13 +18,18 @@ export type RagPipelineSliceShape = {
setNodesDefaultConfigs: (nodesDefaultConfigs: Record<string, any>) => void
ragPipelineVariables: RAGPipelineVariables
setRagPipelineVariables: (ragPipelineVariables: RAGPipelineVariables) => void
dataSourceList: ToolWithProvider[]
setDataSourceList: (dataSourceList: DataSourceItem[]) => void
dataSourceList: RagPipelineDatasourceProviderResponse[]
setDataSourceList: (dataSourceList: RagPipelineDatasourceProviderResponse[]) => void
isPreparingDataSource: boolean
setIsPreparingDataSource: (isPreparingDataSource: boolean) => void
}
export const createRagPipelineSliceSlice: StateCreator<RagPipelineSliceShape> = (set) => ({
export const createRagPipelineSliceSlice: StateCreator<
Partial<RagPipelineSliceShape>,
[],
[],
RagPipelineSliceShape
> = (set) => ({
pipelineId: '',
knowledgeName: '',
showInputFieldPanel: false,
@@ -43,10 +46,7 @@ export const createRagPipelineSliceSlice: StateCreator<RagPipelineSliceShape> =
setRagPipelineVariables: (ragPipelineVariables: RAGPipelineVariables) =>
set(() => ({ ragPipelineVariables })),
dataSourceList: [],
setDataSourceList: (dataSourceList: DataSourceItem[]) => {
const formattedDataSourceList = dataSourceList.map((item) => transformDataSourceToTool(item))
set(() => ({ dataSourceList: formattedDataSourceList }))
},
setDataSourceList: (dataSourceList) => set(() => ({ dataSourceList })),
isPreparingDataSource: false,
setIsPreparingDataSource: (isPreparingDataSource) => set(() => ({ isPreparingDataSource })),
})

View File

@@ -0,0 +1,20 @@
import { resolveDatasourceIcon } from '../datasource-icon'
vi.mock('@/utils/var', () => ({ basePath: '/dify' }))
it.each([
['/console/api/plugin/icon', '/dify/console/api/plugin/icon'],
['/dify/console/api/plugin/icon', '/dify/console/api/plugin/icon'],
['/dify', '/dify'],
['/dify-other/icon.svg', '/dify/dify-other/icon.svg'],
['https://assets.dify.ai/icon.svg', 'https://assets.dify.ai/icon.svg'],
['//assets.dify.ai/icon.svg', '//assets.dify.ai/icon.svg'],
['data:image/svg+xml,icon', 'data:image/svg+xml,icon'],
['icon.svg', 'icon.svg'],
['', ''],
])(
'resolves datasource icon %s without changing external or already prefixed URLs',
(icon, expected) => {
expect(resolveDatasourceIcon(icon)).toBe(expected)
},
)

View File

@@ -0,0 +1,17 @@
import type { RagPipelineDatasourceProviderResponse } from '@dify/contracts/api/console/rag/types.gen'
import { basePath } from '@/utils/var'
export function resolveDatasourceIcon(
icon: RagPipelineDatasourceProviderResponse['declaration']['identity']['icon'],
) {
if (
basePath &&
icon.startsWith('/') &&
!icon.startsWith('//') &&
icon !== basePath &&
!icon.startsWith(`${basePath}/`)
)
return `${basePath}${icon}`
return icon
}

View File

@@ -1,173 +1,227 @@
import type { RagPipelineDatasourceProviderResponse } from '@dify/contracts/api/console/rag/types.gen'
import type { ReactElement } from 'react'
import type { ToolWithProvider } from '../../types'
import type { useMarketplacePlugins } from '@/app/components/plugins/marketplace/query'
import { screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { useMarketplacePlugins } from '@/app/components/plugins/marketplace/query'
import { PluginCategoryEnum } from '@/app/components/plugins/types'
import { useGetLanguage } from '@/context/i18n'
import useTheme from '@/hooks/use-theme'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { renderWithConsoleQuery } from '@/test/console/query-data'
import { Theme } from '@/types/app'
import { BlockEnum } from '../../types'
import { DEFAULT_FILE_EXTENSIONS_IN_LOCAL_FILE_DATA_SOURCE } from '../constants'
import DataSources from '../data-sources'
vi.mock('@/context/i18n', () => ({
useGetLanguage: vi.fn(),
}))
vi.mock('@/hooks/use-theme', () => ({
default: vi.fn(),
}))
vi.mock('@/app/components/plugins/marketplace/query', () => ({
useMarketplacePlugins: vi.fn(),
}))
const mockUseGetLanguage = vi.mocked(useGetLanguage)
const mockUseTheme = vi.mocked(useTheme)
const mockUseMarketplacePlugins = vi.mocked(useMarketplacePlugins)
let enableMarketplaceForRender = false
const render = (ui: ReactElement) =>
renderWithConsoleQuery(ui, {
systemFeatures: { enable_marketplace: enableMarketplaceForRender },
})
type UseMarketplacePluginsReturn = ReturnType<typeof useMarketplacePlugins>
const createToolProvider = (overrides: Partial<ToolWithProvider> = {}): ToolWithProvider => ({
id: 'langgenius/file',
name: 'file',
author: 'Dify',
description: { en_US: 'desc', zh_Hans: '描述' },
icon: 'icon',
label: { en_US: 'File Source', zh_Hans: '文件源' },
type: 'local_file',
team_credentials: {},
is_team_authorization: false,
allow_delete: false,
labels: [],
plugin_id: 'langgenius/file',
meta: { version: '1.0.0' },
tools: [
{
name: 'local-file',
author: 'Dify',
label: { en_US: 'Local File', zh_Hans: '本地文件' },
description: { en_US: 'Load local files', zh_Hans: '加载本地文件' },
parameters: [],
labels: [],
output_schema: {},
},
],
...overrides,
})
const createMarketplacePluginsMock = (
overrides: Partial<UseMarketplacePluginsReturn> = {},
): UseMarketplacePluginsReturn =>
({
const { marketplaceQuery, language, trackEvent } = vi.hoisted(() => ({
marketplaceQuery: vi.fn((_params?: Parameters<typeof useMarketplacePlugins>[0]) => ({
data: undefined,
...overrides,
}) as UseMarketplacePluginsReturn
})),
language: { value: 'en_US' },
trackEvent: vi.fn(),
}))
vi.mock('@/app/components/plugins/marketplace/query', () => ({
useMarketplacePlugins: marketplaceQuery,
}))
vi.mock('@/context/i18n', () => ({ useGetLanguage: () => language.value }))
vi.mock('@/app/components/base/amplitude', () => ({ trackEvent }))
describe('DataSources', () => {
beforeEach(() => {
vi.clearAllMocks()
enableMarketplaceForRender = false
mockUseGetLanguage.mockReturnValue('en_US')
mockUseTheme.mockReturnValue({ theme: Theme.light } as ReturnType<typeof useTheme>)
mockUseMarketplacePlugins.mockReturnValue(createMarketplacePluginsMock())
const render = (ui: ReactElement, enableMarketplace = false) =>
renderWithConsoleQuery(ui, { systemFeatures: { enable_marketplace: enableMarketplace } })
const namedProvider = (name: string): RagPipelineDatasourceProviderResponse => {
const provider = createDatasourceProvider()
return {
...provider,
provider: name,
plugin_id: `dify/${name}`,
declaration: {
...provider.declaration,
identity: { ...provider.declaration.identity, label: { en_US: name } },
},
}
}
beforeEach(() => {
vi.clearAllMocks()
language.value = 'en_US'
})
it('selects the built-in local file datasource directly with its file extensions and canonical provider identity', async () => {
const user = userEvent.setup()
const onSelect = vi.fn()
const provider = createDatasourceProvider()
render(<DataSources searchText="" onSelect={onSelect} dataSources={[provider]} />)
await user.click(screen.getByRole('button', { name: 'File Source' }))
const action = screen.getByRole('button', { name: 'Local File' })
expect(action).toHaveAccessibleDescription('Load local files')
action.focus()
await user.keyboard('{Enter}')
expect(onSelect).toHaveBeenCalledWith(BlockEnum.DataSource, {
plugin_id: 'langgenius/file',
provider_type: 'local_file',
provider_name: 'file',
datasource_name: 'local-file',
datasource_label: 'Local File',
title: 'Local File',
plugin_unique_identifier: 'langgenius/file:1.0.0',
fileExtensions: DEFAULT_FILE_EXTENSIONS_IN_LOCAL_FILE_DATA_SOURCE,
})
// Data source tools should filter by search and normalize the default value payload.
describe('Selection', () => {
it('should add default file extensions for the built-in local file data source', async () => {
const user = userEvent.setup()
const onSelect = vi.fn()
render(<DataSources searchText="" onSelect={onSelect} dataSources={[createToolProvider()]} />)
await user.click(screen.getByText('File Source'))
await user.click(screen.getByText('Local File'))
expect(onSelect).toHaveBeenCalledWith(
BlockEnum.DataSource,
expect.objectContaining({
provider_name: 'file',
datasource_name: 'local-file',
datasource_label: 'Local File',
fileExtensions: expect.arrayContaining(['txt', 'pdf', 'md']),
}),
)
})
it('should filter providers by search text', () => {
render(
<DataSources
searchText="searchable"
onSelect={vi.fn()}
dataSources={[
createToolProvider({
id: 'searchable-provider',
name: 'searchable-provider',
label: { en_US: 'Searchable Source', zh_Hans: '可搜索源' },
tools: [
{
name: 'searchable-tool',
author: 'Dify',
label: { en_US: 'Searchable Tool', zh_Hans: '可搜索工具' },
description: { en_US: 'desc', zh_Hans: '描述' },
parameters: [],
labels: [],
output_schema: {},
},
],
}),
createToolProvider({
id: 'other-provider',
name: 'other-provider',
label: { en_US: 'Other Source', zh_Hans: '其他源' },
}),
]}
/>,
)
expect(screen.getByText('Searchable Source')).toBeInTheDocument()
expect(screen.queryByText('Other Source')).not.toBeInTheDocument()
})
expect(trackEvent).toHaveBeenCalledWith('tool_selected', {
tool_name: 'local-file',
plugin_id: 'langgenius/file',
})
expect(provider.declaration.identity.name).toBe('langgenius/file/file')
expect(screen.queryByRole('button', { name: 'workflow.tabs.addAll' })).not.toBeInTheDocument()
})
// Marketplace search should only run when enabled and a search term is present.
describe('Marketplace Search', () => {
it('should debounce marketplace requests while keeping datasource filtering responsive', async () => {
enableMarketplaceForRender = true
it('keeps unauthorized installed providers selectable and falls back from null locale text', async () => {
language.value = 'zh_Hans'
const user = userEvent.setup()
const provider = namedProvider('drive')
provider.is_authorized = false
provider.declaration.provider_type = 'online_drive'
provider.declaration.identity.label = { en_US: 'Drive source', zh_Hans: null }
provider.declaration.datasources = [
{
identity: {
author: 'Dify',
name: 'drive-pages',
provider: 'dify/drive/drive',
label: { en_US: 'Drive pages', zh_Hans: null },
},
description: { en_US: 'Browse pages', zh_Hans: null },
},
]
const onSelect = vi.fn()
render(<DataSources searchText="pages" onSelect={onSelect} dataSources={[provider]} />)
const onSelect = vi.fn()
const dataSources = [createToolProvider()]
const { rerender } = render(
<DataSources searchText="" onSelect={onSelect} dataSources={dataSources} />,
)
expect(screen.getByRole('button', { name: 'Drive source' })).toHaveAttribute(
'aria-expanded',
'true',
)
await user.click(screen.getByRole('button', { name: 'Drive pages' }))
rerender(<DataSources searchText="i" onSelect={onSelect} dataSources={dataSources} />)
expect(
screen.queryByRole('link', { name: /plugin\.findMoreInMarketplace/ }),
).not.toBeInTheDocument()
rerender(<DataSources searchText="in" onSelect={onSelect} dataSources={dataSources} />)
rerender(<DataSources searchText="invoice" onSelect={onSelect} dataSources={dataSources} />)
expect(
mockUseMarketplacePlugins.mock.calls
.map(([params]) => params)
.filter((params) => params?.query && params.query !== 'invoice'),
).toEqual([])
await waitFor(() => {
expect(mockUseMarketplacePlugins).toHaveBeenLastCalledWith({
query: 'invoice',
category: PluginCategoryEnum.datasource,
})
})
})
expect(onSelect).toHaveBeenCalledWith(BlockEnum.DataSource, {
plugin_id: 'dify/drive',
provider_type: 'online_drive',
provider_name: 'drive',
datasource_name: 'drive-pages',
datasource_label: 'Drive pages',
title: 'Drive pages',
plugin_unique_identifier: provider.plugin_unique_identifier,
})
})
it('filters by raw provider or action name and retains each matching provider group', async () => {
const provider = namedProvider('searchable-provider')
provider.declaration.datasources = [
{
identity: {
author: 'Dify',
name: 'match-action',
provider: 'compound',
label: { en_US: 'Matched action' },
},
description: { en_US: 'Matched description' },
},
{
identity: {
author: 'Dify',
name: 'sibling',
provider: 'compound',
label: { en_US: 'Sibling action' },
},
description: { en_US: '' },
},
]
const { rerender } = render(
<DataSources
searchText="MATCH-action"
onSelect={vi.fn()}
dataSources={[provider, namedProvider('other')]}
/>,
)
expect(screen.getByRole('button', { name: 'Matched action' })).toBeInTheDocument()
expect(screen.getByRole('button', { name: 'Sibling action' })).toBeInTheDocument()
expect(screen.queryByRole('button', { name: 'other' })).not.toBeInTheDocument()
rerender(
<DataSources searchText="searchable-provider" onSelect={vi.fn()} dataSources={[provider]} />,
)
expect(screen.getByRole('button', { name: 'Matched action' })).toBeInTheDocument()
rerender(<DataSources searchText="" onSelect={vi.fn()} dataSources={[provider]} />)
await waitFor(() =>
expect(screen.queryByRole('button', { name: 'Matched action' })).not.toBeInTheDocument(),
)
expect(screen.getByRole('button', { name: 'searchable-provider' })).toHaveAttribute(
'aria-expanded',
'false',
)
})
it('keeps flat letter navigation sorted with pinyin initials and nonalphabetic providers last', () => {
const providers = [
'1Source',
'中文',
'Alpha',
'Echo',
'Foxtrot',
'Gamma',
'Hotel',
'India',
'Juliet',
'Kilo',
'Lima',
].map(namedProvider)
render(<DataSources searchText="" onSelect={vi.fn()} dataSources={providers} />)
const letters = screen
.getAllByRole('button')
.filter((button) => /^[A-Z#]$/.test(button.textContent ?? ''))
expect(letters.map((button) => button.textContent)).toEqual([
'A',
'E',
'F',
'G',
'H',
'I',
'J',
'K',
'L',
'Z',
'#',
])
expect(providers[0]?.provider).toBe('1Source')
})
it('debounces marketplace search while datasource filtering remains immediate', async () => {
const provider = createDatasourceProvider()
const onSelect = vi.fn()
const { rerender } = render(
<DataSources searchText="" onSelect={onSelect} dataSources={[provider]} />,
true,
)
rerender(<DataSources searchText="i" onSelect={onSelect} dataSources={[provider]} />)
rerender(<DataSources searchText="in" onSelect={onSelect} dataSources={[provider]} />)
rerender(<DataSources searchText="invoice" onSelect={onSelect} dataSources={[provider]} />)
expect(screen.queryByRole('button', { name: 'File Source' })).not.toBeInTheDocument()
expect(marketplaceQuery.mock.calls.map(([params]) => params).filter(Boolean)).toEqual([])
await waitFor(() =>
expect(marketplaceQuery).toHaveBeenLastCalledWith({ query: 'invoice', category: 'datasource' }),
)
})
it('keeps an expanded datasource open when the language changes its sort letter', async () => {
const user = userEvent.setup()
const provider = createDatasourceProvider()
const { rerender } = render(
<DataSources searchText="" onSelect={vi.fn()} dataSources={[provider]} />,
)
await user.click(screen.getByRole('button', { name: 'File Source' }))
expect(screen.getByRole('button', { name: 'Local File' })).toBeInTheDocument()
language.value = 'zh_Hans'
rerender(<DataSources searchText="" onSelect={vi.fn()} dataSources={[provider]} />)
expect(screen.getByRole('button', { name: '文件源' })).toHaveAttribute('aria-expanded', 'true')
expect(screen.getByRole('button', { name: '本地文件' })).toBeInTheDocument()
})

View File

@@ -67,12 +67,9 @@ describe('createToolListData', () => {
])
})
it('maps workflow and data source providers to distinct categories', () => {
it('maps workflow providers to their own category', () => {
const result = createToolListData(
[
createToolProvider({ id: 'workflow', type: CollectionType.workflow }),
createToolProvider({ id: 'data-source', type: 'local_file' }),
],
[createToolProvider({ id: 'workflow', type: CollectionType.workflow })],
() => 'A',
)
@@ -82,11 +79,6 @@ describe('createToolListData', () => {
category: 'workflow',
tools: [expect.objectContaining({ id: 'workflow' })],
},
{
kind: 'category',
category: 'data-source',
tools: [expect.objectContaining({ id: 'data-source' })],
},
])
})

View File

@@ -1,109 +0,0 @@
import type { DataSourceItem } from '../types'
import { transformDataSourceToTool } from '../utils'
const createLocalizedText = (text: string) => ({
en_US: text,
zh_Hans: text,
})
const createDataSourceItem = (overrides: Partial<DataSourceItem> = {}): DataSourceItem => ({
plugin_id: 'plugin-1',
plugin_unique_identifier: 'plugin-1@provider',
provider: 'provider-a',
declaration: {
credentials_schema: [{ name: 'api_key' }],
provider_type: 'local_file',
identity: {
author: 'Dify',
description: createLocalizedText('Datasource provider'),
icon: 'provider-icon',
label: createLocalizedText('Provider A'),
name: 'provider-a',
tags: ['retrieval', 'storage'],
},
datasources: [
{
description: createLocalizedText('Search in documents'),
identity: {
author: 'Dify',
label: createLocalizedText('Document Search'),
name: 'document_search',
provider: 'provider-a',
},
parameters: [{ name: 'query', type: 'string' }],
output_schema: {
type: 'object',
properties: {
result: { type: 'string' },
},
},
},
],
},
is_authorized: true,
...overrides,
})
describe('transformDataSourceToTool', () => {
it('should map datasource provider fields to tool shape', () => {
const dataSourceItem = createDataSourceItem()
const result = transformDataSourceToTool(dataSourceItem)
expect(result).toMatchObject({
id: 'plugin-1',
provider: 'provider-a',
name: 'provider-a',
author: 'Dify',
description: createLocalizedText('Datasource provider'),
icon: 'provider-icon',
label: createLocalizedText('Provider A'),
type: 'local_file',
allow_delete: true,
is_authorized: true,
is_team_authorization: true,
labels: ['retrieval', 'storage'],
plugin_id: 'plugin-1',
plugin_unique_identifier: 'plugin-1@provider',
credentialsSchema: [{ name: 'api_key' }],
meta: { version: '' },
})
expect(result.team_credentials).toEqual({})
expect(result.tools).toEqual([
{
name: 'document_search',
author: 'Dify',
label: createLocalizedText('Document Search'),
description: createLocalizedText('Search in documents'),
parameters: [{ name: 'query', type: 'string' }],
labels: [],
output_schema: {
type: 'object',
properties: {
result: { type: 'string' },
},
},
},
])
})
it('should fallback to empty arrays when tags and credentials schema are missing', () => {
const baseDataSourceItem = createDataSourceItem()
const dataSourceItem = createDataSourceItem({
declaration: {
...baseDataSourceItem.declaration,
credentials_schema:
undefined as unknown as DataSourceItem['declaration']['credentials_schema'],
identity: {
...baseDataSourceItem.declaration.identity,
tags: undefined as unknown as DataSourceItem['declaration']['identity']['tags'],
},
},
})
const result = transformDataSourceToTool(dataSourceItem)
expect(result.labels).toEqual([])
expect(result.credentialsSchema).toEqual([])
})
})

View File

@@ -1,27 +1,31 @@
import type { OnSelectBlock, ToolWithProvider } from '../types'
import type { DataSourceDefaultValue, ToolDefaultValue } from './types'
import type {
DatasourceEntity,
RagPipelineDatasourceProviderResponse,
} from '@dify/contracts/api/console/rag/types.gen'
import type { OnSelectBlock } from '../types'
import type { DataSourceDefaultValue } from './types'
import type { ListRef } from '@/app/components/workflow/block-selector/marketplace-plugin/list'
import { zDatasourceProviderType } from '@dify/contracts/api/console/workspaces/zod.gen'
import { cn } from '@langgenius/dify-ui/cn'
import { useSuspenseQuery } from '@tanstack/react-query'
import { useDebounce } from 'ahooks'
import { useCallback, useMemo, useRef } from 'react'
import { trackEvent } from '@/app/components/base/amplitude'
import { useMarketplacePlugins } from '@/app/components/plugins/marketplace/query'
import PluginList from '@/app/components/workflow/block-selector/marketplace-plugin/list'
import { useGetLanguage } from '@/context/i18n'
import { systemFeaturesQueryOptions } from '@/features/system-features/client'
import { renderI18nObject } from '@/i18n/metadata'
import { PluginCategoryEnum } from '../../plugins/types'
import { BlockEnum } from '../types'
import { DEFAULT_FILE_EXTENSIONS_IN_LOCAL_FILE_DATA_SOURCE } from './constants'
import Tools from './tools'
import { ViewType } from './types'
import { DatasourceList } from './datasource-list'
type DataSourcesProps = {
className?: string
toolContentClassName?: string
searchText: string
onSelect: OnSelectBlock
dataSources: ToolWithProvider[]
dataSources: RagPipelineDatasourceProviderResponse[]
}
function DataSources({
@@ -40,46 +44,42 @@ function DataSources({
}
const filteredDatasources = useMemo(() => {
const hasFilter = searchText
if (!hasFilter)
return dataSources.filter((toolWithProvider) => toolWithProvider.tools.length > 0)
return dataSources.filter((toolWithProvider) => {
return dataSources.filter((provider) => {
const datasources = provider.declaration.datasources ?? []
if (!searchText) return datasources.length > 0
return (
isMatchingKeywords(toolWithProvider.name, searchText) ||
toolWithProvider.tools.some((tool) => {
return (
tool.label[language]!.toLowerCase().includes(searchText.toLowerCase()) ||
tool.name.toLowerCase().includes(searchText.toLowerCase())
)
})
isMatchingKeywords(provider.provider, searchText) ||
datasources.some(
(datasource) =>
isMatchingKeywords(renderI18nObject(datasource.identity.label, language), searchText) ||
isMatchingKeywords(datasource.identity.name, searchText),
)
)
})
}, [searchText, dataSources, language])
const handleSelect = useCallback(
(_: BlockEnum, toolDefaultValue: ToolDefaultValue) => {
let defaultValue: DataSourceDefaultValue = {
plugin_id: toolDefaultValue?.provider_id,
provider_type: zDatasourceProviderType.parse(toolDefaultValue.provider_type),
provider_name: toolDefaultValue?.provider_name,
datasource_name: toolDefaultValue?.tool_name,
datasource_label: toolDefaultValue?.tool_label,
title: toolDefaultValue?.title,
plugin_unique_identifier: toolDefaultValue?.plugin_unique_identifier,
(provider: RagPipelineDatasourceProviderResponse, datasource: DatasourceEntity) => {
const label = renderI18nObject(datasource.identity.label, language)
const defaultValue: DataSourceDefaultValue = {
plugin_id: provider.plugin_id,
provider_type: provider.declaration.provider_type,
provider_name: provider.provider,
datasource_name: datasource.identity.name,
datasource_label: label,
title: label,
plugin_unique_identifier: provider.plugin_unique_identifier,
...(provider.plugin_id === 'langgenius/file' && provider.provider === 'file'
? { fileExtensions: DEFAULT_FILE_EXTENSIONS_IN_LOCAL_FILE_DATA_SOURCE }
: {}),
}
if (
toolDefaultValue?.provider_id === 'langgenius/file' &&
toolDefaultValue?.provider_name === 'file'
) {
defaultValue = {
...defaultValue,
fileExtensions: DEFAULT_FILE_EXTENSIONS_IN_LOCAL_FILE_DATA_SOURCE,
}
}
onSelect(BlockEnum.DataSource, toolDefaultValue && defaultValue)
onSelect(BlockEnum.DataSource, defaultValue)
trackEvent('tool_selected', {
tool_name: datasource.identity.name,
plugin_id: provider.plugin_id,
})
},
[onSelect],
[onSelect, language],
)
const { data: enable_marketplace } = useSuspenseQuery({
@@ -118,13 +118,11 @@ function DataSources({
className="max-h-116 overflow-x-hidden overflow-y-auto"
onScroll={() => pluginRef.current?.handleScroll()}
>
<Tools
<DatasourceList
className={toolContentClassName}
tools={filteredDatasources}
onSelect={handleSelect as OnSelectBlock}
viewType={ViewType.flat}
providers={filteredDatasources}
onSelect={handleSelect}
hasSearchText={!!searchText}
canNotSelectMultiple
/>
{enable_marketplace && (
<PluginList

View File

@@ -0,0 +1,219 @@
import type {
DatasourceEntity,
RagPipelineDatasourceProviderResponse,
} from '@dify/contracts/api/console/rag/types.gen'
import type { PreviewCardHandle } from '@langgenius/dify-ui/preview-card'
import { Button } from '@langgenius/dify-ui/button'
import { cn } from '@langgenius/dify-ui/cn'
import { Collapsible, CollapsiblePanel, CollapsibleTrigger } from '@langgenius/dify-ui/collapsible'
import {
createPreviewCardHandle,
PreviewCard,
PreviewCardTrigger,
} from '@langgenius/dify-ui/preview-card'
import { useId, useMemo, useRef, useState } from 'react'
import { resolveDatasourceIcon } from '@/app/components/rag-pipeline/utils/datasource-icon'
import Empty from '@/app/components/tools/provider/empty'
import { useGetLanguage } from '@/context/i18n'
import { renderI18nObject } from '@/i18n/metadata'
import BlockIcon from '../block-icon'
import { BlockEnum } from '../types'
import { IndexBar } from './index-bar'
import { BlockSelectorPreviewCardContent } from './preview-card'
import { compareProviderLetters, getProviderLetter } from './utils'
type DatasourcePreview = { icon: string; label: string; description: string }
type OnSelectDatasource = (
provider: RagPipelineDatasourceProviderResponse,
datasource: DatasourceEntity,
) => void
function DatasourceAction({
provider,
datasource,
previewCardHandle,
onSelect,
}: {
provider: RagPipelineDatasourceProviderResponse
datasource: DatasourceEntity
previewCardHandle: PreviewCardHandle<DatasourcePreview>
onSelect: OnSelectDatasource
}) {
const language = useGetLanguage()
const descriptionId = useId()
const label = renderI18nObject(datasource.identity.label, language)
const description = renderI18nObject(datasource.description, language)
return (
<>
<PreviewCardTrigger
delay={150}
closeDelay={150}
handle={previewCardHandle}
payload={{
icon: resolveDatasourceIcon(provider.declaration.identity.icon),
label,
description,
}}
render={
<Button
variant="ghost"
size="medium"
aria-describedby={description ? descriptionId : undefined}
className="w-full justify-between pr-1 pl-5.25 text-left focus-visible:ring-inset disabled:cursor-default"
onClick={() => onSelect(provider, datasource)}
>
<div className="truncate border-l-2 border-divider-subtle py-2 pl-4 system-sm-medium text-text-secondary">
<span>{label}</span>
</div>
</Button>
}
/>
{description && (
<span id={descriptionId} className="sr-only">
{description}
</span>
)}
</>
)
}
function DatasourceGroup({
provider,
hasSearchText,
previewCardHandle,
onSelect,
}: {
provider: RagPipelineDatasourceProviderResponse
hasSearchText: boolean
previewCardHandle: PreviewCardHandle<DatasourcePreview>
onSelect: OnSelectDatasource
}) {
const language = useGetLanguage()
const panelId = useId()
const [open, setOpen] = useState(hasSearchText)
const [previousHasSearchText, setPreviousHasSearchText] = useState(hasSearchText)
if (previousHasSearchText !== hasSearchText) {
setPreviousHasSearchText(hasSearchText)
setOpen(hasSearchText)
}
return (
<Collapsible className="mb-1 last-of-type:mb-0" open={open} onOpenChange={setOpen}>
<div className="group/item relative flex w-full items-center rounded-lg">
<CollapsibleTrigger
aria-controls={panelId}
className="group/collapsible flex h-8 min-h-8 w-full min-w-0 touch-manipulation items-center justify-start gap-0 rounded-lg bg-transparent pr-2 pl-3 text-start system-sm-medium text-text-secondary outline-hidden select-none group-hover/item:bg-state-base-hover hover:bg-state-base-hover hover:text-text-primary focus-visible:ring-2 focus-visible:ring-state-accent-solid focus-visible:ring-inset data-panel-open:text-text-primary"
>
<div className="flex h-8 min-w-0 grow items-center">
<BlockIcon
className="shrink-0"
type={BlockEnum.DataSource}
toolIcon={resolveDatasourceIcon(provider.declaration.identity.icon)}
/>
<div className="ml-2 flex w-0 grow items-center text-sm text-text-primary">
<span className="max-w-62.5 truncate">
{renderI18nObject(provider.declaration.identity.label, language)}
</span>
</div>
</div>
<span
aria-hidden
className="ml-2 i-ri-arrow-right-s-line size-4 shrink-0 text-text-quaternary transition-transform group-data-panel-open/collapsible:rotate-90 motion-reduce:transition-none"
/>
</CollapsibleTrigger>
</div>
<CollapsiblePanel id={panelId}>
{provider.declaration.datasources?.map((datasource) => (
<DatasourceAction
key={datasource.identity.name}
provider={provider}
datasource={datasource}
previewCardHandle={previewCardHandle}
onSelect={onSelect}
/>
))}
</CollapsiblePanel>
</Collapsible>
)
}
export function DatasourceList({
providers,
hasSearchText,
onSelect,
className,
}: {
providers: RagPipelineDatasourceProviderResponse[]
hasSearchText: boolean
onSelect: OnSelectDatasource
className?: string
}) {
const language = useGetLanguage()
const [previewCardHandle] = useState(() => createPreviewCardHandle<DatasourcePreview>())
const itemRefsRef = useRef<Record<string, HTMLDivElement | null>>({})
const groups = useMemo(() => {
const buckets = new Map<string, RagPipelineDatasourceProviderResponse[]>()
for (const provider of providers) {
const label =
renderI18nObject(provider.declaration.identity.label, language) || provider.provider
const letter = getProviderLetter(label[0] ?? '#')
const bucket = buckets.get(letter) ?? []
bucket.push(provider)
buckets.set(letter, bucket)
}
return [...buckets.entries()].sort(([left], [right]) => compareProviderLetters(left, right))
}, [providers, language])
return (
<div className={cn('max-w-full p-1', className)}>
{!providers.length && !hasSearchText && (
<div className="py-10">
<Empty />
</div>
)}
<div className="flex w-full min-w-0">
<div className="mr-1 min-w-0 grow">
{groups.flatMap(([letter, group]) =>
group.map((provider, index) => (
<div
key={`${provider.plugin_id}/${provider.provider}`}
ref={(element) => {
if (index === 0) itemRefsRef.current[letter] = element
}}
>
<DatasourceGroup
provider={provider}
hasSearchText={hasSearchText}
onSelect={onSelect}
previewCardHandle={previewCardHandle}
/>
</div>
)),
)}
</div>
{providers.length > 10 && (
<IndexBar letters={groups.map(([letter]) => letter)} itemRefs={itemRefsRef} />
)}
</div>
<PreviewCard handle={previewCardHandle}>
{({ payload }) =>
payload && (
<BlockSelectorPreviewCardContent>
<BlockIcon
size="md"
className="mb-2"
type={BlockEnum.DataSource}
toolIcon={payload.icon}
/>
<div className="mb-1 text-sm/5 wrap-break-word text-text-primary">
{payload.label}
</div>
<div className="text-xs leading-4.5 wrap-break-word text-text-secondary">
{payload.description}
</div>
</BlockSelectorPreviewCardContent>
)
}
</PreviewCard>
</div>
)
}

View File

@@ -1,3 +1,4 @@
import type { RagPipelineDatasourceProviderResponse } from '@dify/contracts/api/console/rag/types.gen'
import type {
PopoverPopupProps,
PopoverPositionerProps,
@@ -5,13 +6,7 @@ import type {
PopoverTriggerProps,
} from '@langgenius/dify-ui/popover'
import type { CSSProperties, KeyboardEvent, MouseEventHandler, Ref } from 'react'
import type {
CommonNodeType,
NodeDefault,
OnNodeAdd,
OnSelectBlock,
ToolWithProvider,
} from '../types'
import type { CommonNodeType, NodeDefault, OnNodeAdd, OnSelectBlock } from '../types'
import type { TabType } from './types'
import { cn } from '@langgenius/dify-ui/cn'
import { IconButton } from '@langgenius/dify-ui/icon-button'
@@ -56,7 +51,7 @@ export type BlockSelectorProps = Pick<
availableBlocksTypes?: BlockEnum[]
disabled?: boolean
blocks?: NodeDefault[]
dataSources?: ToolWithProvider[]
dataSources?: RagPipelineDatasourceProviderResponse[]
noBlocks?: boolean
noTools?: boolean
standalonePanel?: TabType

View File

@@ -1,5 +1,6 @@
import type { RagPipelineDatasourceProviderResponse } from '@dify/contracts/api/console/rag/types.gen'
import type { ReactNode, Ref } from 'react'
import type { BlockEnum, NodeDefault, OnNodeAdd, OnSelectBlock, ToolWithProvider } from '../types'
import type { BlockEnum, NodeDefault, OnNodeAdd, OnSelectBlock } from '../types'
import { cn } from '@langgenius/dify-ui/cn'
import { Tabs, TabsIndicator, TabsList, TabsPanel, TabsTab } from '@langgenius/dify-ui/tabs'
import { Tooltip, TooltipContent, TooltipTrigger } from '@langgenius/dify-ui/tooltip'
@@ -30,7 +31,7 @@ type BlockSelectorPanelsProps = {
onRequestClose: () => void
availableBlocksTypes?: BlockEnum[]
blocks: NodeDefault[]
dataSources?: ToolWithProvider[]
dataSources?: RagPipelineDatasourceProviderResponse[]
allowStartNodeSelection?: boolean
hasUserInputNode?: boolean
hasTriggerNode?: boolean

View File

@@ -1,16 +1,8 @@
import type { DatasourceProviderType } from '@dify/contracts/api/console/workspaces/types.gen'
import type { ToolWithProvider } from '../types'
import { CollectionType } from '../../tools/types'
import { compareProviderLetters, getProviderLetter } from './utils'
type ToolCategoryGroup = 'custom' | 'data-source' | 'mcp' | 'workflow'
const datasourceProviderTypes: Record<DatasourceProviderType, true> = {
local_file: true,
online_document: true,
online_drive: true,
website_crawl: true,
}
type ToolCategoryGroup = 'custom' | 'mcp' | 'workflow'
type AuthorToolGroup = {
kind: 'author'
@@ -111,14 +103,9 @@ function addToolToAuthorGroup(bucket: LetterBucket, tool: ToolWithProvider) {
function getToolCategoryGroup(type: ToolWithProvider['type']): ToolCategoryGroup | undefined {
if (type === CollectionType.custom) return 'custom'
if (type === CollectionType.workflow) return 'workflow'
if (isDatasourceProviderType(type)) return 'data-source'
if (type === CollectionType.mcp) return 'mcp'
}
function isDatasourceProviderType(type: ToolWithProvider['type']): type is DatasourceProviderType {
return Object.hasOwn(datasourceProviderTypes, type)
}
function mergeGroupsByProvider(buckets: LetterBucket[]) {
const groups: ToolGroup[] = []
const authorGroups = new Map<string, AuthorToolGroup>()

View File

@@ -1,3 +1,4 @@
import type { RagPipelineDatasourceProviderResponse } from '@dify/contracts/api/console/rag/types.gen'
import type { OnSelectBlock, ToolWithProvider } from '../types'
import { useSuspenseQuery } from '@tanstack/react-query'
import { useEffect, useMemo } from 'react'
@@ -78,7 +79,7 @@ export function ToolPanel({
tags: string[]
onTagsChange: (tags: string[]) => void
onSelect: OnSelectBlock
dataSources: ToolWithProvider[]
dataSources: RagPipelineDatasourceProviderResponse[]
}) {
const { data: buildInTools } = useAllBuiltInTools()
const { data: customTools } = useAllCustomTools()

View File

@@ -77,17 +77,6 @@ describe('ToolListTreeView', () => {
}),
],
},
{
kind: 'category',
category: 'data-source',
tools: [
createToolProvider({
id: 'data-source-provider',
type: 'local_file',
label: { en_US: 'Data Source Provider', zh_Hans: 'Data Source Provider' },
}),
],
},
]}
previewCardHandle={createPreviewCardHandle()}
hasSearchText={false}
@@ -99,7 +88,6 @@ describe('ToolListTreeView', () => {
expect(screen.getByText('workflow.tabs.customTool')).toBeInTheDocument()
expect(screen.getByText('MCP')).toBeInTheDocument()
expect(screen.getByText('workflow.tabs.workflowTool')).toBeInTheDocument()
expect(screen.getByText('workflow.tabs.sources')).toBeInTheDocument()
expect(screen.getByText('Built In Provider')).toBeInTheDocument()
expect(screen.getByText('Custom Provider')).toBeInTheDocument()
expect(screen.getByText('MCP Provider')).toBeInTheDocument()

View File

@@ -36,8 +36,6 @@ export const ToolListTreeView = memo(
return t(($) => $['tabs.customTool'], { ns: 'workflow' })
case 'workflow':
return t(($) => $['tabs.workflowTool'], { ns: 'workflow' })
case 'data-source':
return t(($) => $['tabs.sources'], { ns: 'workflow' })
case 'mcp':
return 'MCP'
}

View File

@@ -1,5 +1,5 @@
import type { AgentInviteOptionResponse } from '@dify/contracts/api/console/agent/types.gen'
import type { DatasourceProviderType } from '@dify/contracts/api/console/workspaces/types.gen'
import type { RagPipelineDatasourceProviderResponse } from '@dify/contracts/api/console/rag/types.gen'
import type {
ParametersSchema,
PluginMeta,
@@ -86,7 +86,8 @@ export type ToolDefaultValue = PluginCommonDefaultValue & {
plugin_unique_identifier?: string
}
export type DataSourceDefaultValue = Omit<PluginCommonDefaultValue, 'provider_id'> & {
export type DataSourceDefaultValue = {
provider_type: RagPipelineDatasourceProviderResponse['declaration']['provider_type']
plugin_id: string
provider_name: string
datasource_name: string
@@ -139,40 +140,6 @@ export type ToolValue = {
type?: string
}
export type DataSourceItem = {
plugin_id: string
plugin_unique_identifier: string
provider: string
declaration: {
credentials_schema: unknown[]
provider_type: DatasourceProviderType
identity: {
author: string
description: TypeWithI18N
icon: string | { background: string; content: string }
label: TypeWithI18N
name: string
tags: string[]
}
datasources: {
description: TypeWithI18N
identity: {
author: string
icon?: string | { background: string; content: string }
label: TypeWithI18N
name: string
provider: string
}
parameters: unknown[]
output_schema?: {
type: string
properties: Record<string, unknown>
}
}[]
}
is_authorized: boolean
}
type TriggerCredentialField = {
type:
| 'secret-input'

View File

@@ -1,42 +1,5 @@
import type { DataSourceItem } from './types'
import type { Tool } from '@/app/components/tools/types'
import { pinyin } from 'pinyin-pro'
export const transformDataSourceToTool = (dataSourceItem: DataSourceItem) => {
return {
id: dataSourceItem.plugin_id,
provider: dataSourceItem.provider,
name: dataSourceItem.provider,
author: dataSourceItem.declaration.identity.author,
description: dataSourceItem.declaration.identity.description,
icon: dataSourceItem.declaration.identity.icon,
label: dataSourceItem.declaration.identity.label,
type: dataSourceItem.declaration.provider_type,
team_credentials: {},
allow_delete: true,
is_team_authorization: dataSourceItem.is_authorized,
is_authorized: dataSourceItem.is_authorized,
labels: dataSourceItem.declaration.identity.tags || [],
plugin_id: dataSourceItem.plugin_id,
plugin_unique_identifier: dataSourceItem.plugin_unique_identifier,
tools: dataSourceItem.declaration.datasources.map((datasource) => {
return {
name: datasource.identity.name,
author: datasource.identity.author,
label: datasource.identity.label,
description: datasource.description,
parameters: datasource.parameters,
labels: [],
output_schema: datasource.output_schema,
} as Tool
}),
credentialsSchema: dataSourceItem.declaration.credentials_schema || [],
meta: {
version: '',
},
}
}
export function getProviderLetter(firstChar: string) {
const pinyinInitial = /[\u4E00-\u9FA5]/.test(firstChar)
? pinyin(firstChar, { pattern: 'first', toneType: 'none' })[0]

View File

@@ -3,8 +3,8 @@ import type {
BlockEnum,
Node,
NodeDefault,
ToolWithProvider,
ValueSelector,
WorkflowPluginCatalogs,
} from '@/app/components/workflow/types'
import type { IOtherOptions } from '@/service/base'
import type { SchemaTypeDefinition } from '@/service/use-common'
@@ -78,7 +78,7 @@ type CommonHooksFnMap = {
fetchInspectVars: (params: {
passInVars?: boolean
vars?: VarInInspect[]
passedInAllPluginInfoList?: Record<string, ToolWithProvider[]>
passedInAllPluginInfoList?: WorkflowPluginCatalogs
passedInSchemaTypeDefinitions?: SchemaTypeDefinition[]
}) => Promise<void>
hasNodeInspectVars: (nodeId: string) => boolean

View File

@@ -1,7 +1,11 @@
import type { CommonNodeType } from '../../types'
import type { CollectionProviderType } from '@/app/components/tools/types'
import { QueryClient } from '@tanstack/react-query'
import { act } from '@testing-library/react'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { CollectionType } from '@/app/components/tools/types'
import { consoleQuery } from '@/service/console'
import { seedAccountProfileQuery } from '@/test/console/account-profile'
import { renderWorkflowHook } from '../../__tests__/workflow-test-env'
import { BlockEnum } from '../../types'
import { useNodePluginInstallation } from '../use-node-plugin-installation'
@@ -13,7 +17,6 @@ const mockMcpTools = vi.fn()
const mockInvalidToolsByType = vi.fn()
const mockTriggerPlugins = vi.fn()
const mockInvalidateTriggers = vi.fn()
const mockInvalidDataSourceList = vi.fn()
let mockWorkspacePermissionKeys = ['plugin.install']
vi.mock('@/context/permission-state', async () => {
@@ -37,10 +40,6 @@ vi.mock('@/service/use-triggers', () => ({
useInvalidateAllTriggerPlugins: () => mockInvalidateTriggers,
}))
vi.mock('@/service/use-pipeline', () => ({
useInvalidDataSourceList: () => mockInvalidDataSourceList,
}))
const makeToolNode = (overrides: Partial<CommonNodeType> = {}) =>
({
type: BlockEnum.Tool,
@@ -90,11 +89,11 @@ const matchedTriggerProvider = {
plugin_id: 'trigger-plugin',
}
const matchedDataSource = {
const matchedDataSource = createDatasourceProvider({
provider: 'knowledge-provider',
plugin_id: 'knowledge-plugin',
plugin_unique_identifier: 'knowledge-plugin@1.0.0',
}
})
describe('useNodePluginInstallation', () => {
beforeEach(() => {
@@ -107,7 +106,6 @@ describe('useNodePluginInstallation', () => {
mockInvalidToolsByType.mockReturnValue(undefined)
mockTriggerPlugins.mockReturnValue({ data: undefined, isLoading: false })
mockInvalidateTriggers.mockReset()
mockInvalidDataSourceList.mockReset()
})
it('should return the noop installation state for non plugin-dependent nodes', () => {
@@ -249,7 +247,7 @@ describe('useNodePluginInstallation', () => {
),
{
initialStoreState: {
dataSourceList: [matchedDataSource] as never,
dataSourceList: [matchedDataSource],
},
},
)
@@ -258,11 +256,18 @@ describe('useNodePluginInstallation', () => {
expect(missingRender.result.current.isMissing).toBe(true)
expect(missingRender.result.current.shouldDim).toBe(true)
const queryClient = new QueryClient({
defaultOptions: { queries: { staleTime: Infinity, retry: false } },
})
seedAccountProfileQuery(queryClient)
const catalogKey = consoleQuery.rag.pipelines.datasourcePlugins.get.queryKey()
queryClient.setQueryData(catalogKey, [matchedDataSource])
const matchedRender = renderWorkflowHook(
() => useNodePluginInstallation(makeDataSourceNode()),
{
queryClient,
initialStoreState: {
dataSourceList: [matchedDataSource] as never,
dataSourceList: [matchedDataSource],
},
},
)
@@ -274,7 +279,8 @@ describe('useNodePluginInstallation', () => {
matchedRender.result.current.onInstallSuccess()
})
expect(mockInvalidDataSourceList).toHaveBeenCalled()
expect(queryClient.getQueryState(catalogKey)?.isInvalidated).toBe(true)
queryClient.clear()
})
it('should keep data sources in checking state before the list is loaded', () => {

View File

@@ -1,4 +1,5 @@
import type { Node } from '../../types'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { CollectionType } from '@/app/components/tools/types'
import { renderWorkflowHook } from '../../__tests__/workflow-test-env'
import { BlockEnum } from '../../types'
@@ -109,13 +110,18 @@ describe('useNodesMetaData', () => {
},
})
const datasource = {
const provider = createDatasourceProvider()
const datasource = createDatasourceProvider({
plugin_id: 'datasource-1',
author: 'Datasource Author',
description: {
'en-US': 'Datasource description',
declaration: {
...provider.declaration,
identity: {
...provider.declaration.identity,
author: 'Datasource Author',
description: { en_US: 'Datasource description', zh_Hans: null },
},
},
}
})
const metadataMap = {
[BlockEnum.LLM]: {
@@ -130,7 +136,7 @@ describe('useNodesMetaData', () => {
const datasourceResult = renderWorkflowHook(() => useNodeMetaData(datasourceNode), {
initialStoreState: {
dataSourceList: [datasource as never],
dataSourceList: [datasource],
},
hooksStoreProps: {
availableNodesMetaData: {

View File

@@ -1,3 +1,4 @@
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { CollectionType } from '@/app/components/tools/types'
import { resetReactFlowMockState } from '../../__tests__/reactflow-mock-state'
import { renderWorkflowHook } from '../../__tests__/workflow-test-env'
@@ -38,6 +39,11 @@ vi.mock('@/hooks/use-theme', () => ({
default: () => ({ theme: mockTheme }),
}))
vi.mock('@/utils/var', async (importOriginal) => ({
...(await importOriginal<typeof import('@/utils/var')>()),
basePath: '/dify',
}))
vi.mock('@/utils', () => ({
canFindTool: (id: string, target: string) => id === target,
}))
@@ -150,12 +156,10 @@ describe('useToolIcon', () => {
expect(
renderWorkflowHook(() => useToolIcon(dataSourceData), {
initialStoreState: {
dataSourceList: [
{ id: 'ds-1', plugin_id: 'datasource-1', icon: '/datasource.svg' },
] as never,
dataSourceList: [createDatasourceProvider({ plugin_id: 'datasource-1' })],
},
}).result.current,
).toBe('/datasource.svg')
).toBe('/dify/datasource.svg')
})
it('should fallback to provider_icon when no collection match', () => {
@@ -237,9 +241,7 @@ describe('useGetToolIcon', () => {
buildInTools: [
{ id: 'override-1', name: 'override', icon: '/override.svg', plugin_id: 'p1' },
] as never,
dataSourceList: [
{ id: 'ds-1', plugin_id: 'datasource-1', icon: '/datasource-store.svg' },
] as never,
dataSourceList: [createDatasourceProvider({ plugin_id: 'datasource-1' })],
},
})
@@ -258,7 +260,7 @@ describe('useGetToolIcon', () => {
type: BlockEnum.DataSource,
plugin_id: 'datasource-1',
}),
).toBe('/datasource-store.svg')
).toBe('/dify/datasource.svg')
expect(store.getState().buildInTools).toHaveLength(1)
})

View File

@@ -1,4 +1,4 @@
import type { Node, ToolWithProvider } from '@/app/components/workflow/types'
import type { Node, WorkflowPluginCatalogs } from '@/app/components/workflow/types'
import type { SchemaTypeDefinition } from '@/service/use-common'
import type { FlowType } from '@/types/common'
import type { NodeWithVar, VarInInspect } from '@/types/workflow'
@@ -51,7 +51,7 @@ export const useSetWorkflowVarsWithValue = ({ flowType, flowId }: Params) => {
const setInspectVarsToStore = useCallback(
(
inspectVars: VarInInspect[],
passedInAllPluginInfoList?: Record<string, ToolWithProvider[]>,
passedInAllPluginInfoList?: WorkflowPluginCatalogs,
passedInSchemaTypeDefinitions?: SchemaTypeDefinition[],
) => {
const { setNodesWithInspectVars } = workflowStore.getState()
@@ -121,7 +121,7 @@ export const useSetWorkflowVarsWithValue = ({ flowType, flowId }: Params) => {
async (params: {
passInVars?: boolean
vars?: VarInInspect[]
passedInAllPluginInfoList?: Record<string, ToolWithProvider[]>
passedInAllPluginInfoList?: WorkflowPluginCatalogs
passedInSchemaTypeDefinitions?: SchemaTypeDefinition[]
}) => {
const { passInVars, vars, passedInAllPluginInfoList, passedInSchemaTypeDefinitions } = params

View File

@@ -2,10 +2,11 @@ import type { DataSourceNodeType } from '../nodes/data-source/types'
import type { ToolNodeType } from '../nodes/tool/types'
import type { PluginTriggerNodeType } from '../nodes/trigger-plugin/types'
import type { CommonNodeType } from '../types'
import { useQueryClient } from '@tanstack/react-query'
import { useCallback, useMemo } from 'react'
import useWorkspacePluginInstallPermission from '@/app/components/plugins/install-plugin/hooks/use-workspace-plugin-install-permission'
import { CollectionType } from '@/app/components/tools/types'
import { useInvalidDataSourceList } from '@/service/use-pipeline'
import { consoleQuery } from '@/service/console'
import {
useAllBuiltInTools,
useAllCustomTools,
@@ -168,8 +169,8 @@ const useDataSourceInstallation = (
_enabled: boolean,
canInstallPlugin: boolean,
): InstallationState => {
const queryClient = useQueryClient()
const dataSourceList = useStore((s) => s.dataSourceList)
const invalidateDataSourceList = useInvalidDataSourceList()
const { plugin_unique_identifier, plugin_id, provider_name } = data
const matchedPlugin = useMemo(() => {
@@ -182,8 +183,10 @@ const useDataSourceInstallation = (
const canInstall = Boolean(data.plugin_unique_identifier) && canInstallPlugin
const onInstallSuccess = useCallback(() => {
invalidateDataSourceList()
}, [invalidateDataSourceList])
queryClient.invalidateQueries({
queryKey: consoleQuery.rag.pipelines.datasourcePlugins.get.key(),
})
}, [queryClient])
const hasLoadedList = dataSourceList !== undefined

View File

@@ -7,8 +7,10 @@ import { useStore } from '@/app/components/workflow/store'
import { BlockEnum } from '@/app/components/workflow/types'
import { getNodeCatalogType } from '@/app/components/workflow/utils/node'
import { useGetLanguage } from '@/context/i18n'
import { renderI18nObject } from '@/i18n/metadata'
import { useAllBuiltInTools, useAllCustomTools, useAllWorkflowTools } from '@/service/use-tools'
import { matchesProviderReference } from '@/utils/provider-reference'
import { matchDataSource } from '../utils/plugin-install-check'
export const useNodesMetaData = () => {
const availableNodesMetaData = useHooksStore((s) => s.availableNodesMetaData)
@@ -32,7 +34,7 @@ export const useNodeMetaData = (node: Node) => {
const nodeMetaData = availableNodesMetaData.nodesMap?.[getNodeCatalogType(data)]
const author = useMemo(() => {
if (data.type === BlockEnum.DataSource)
return dataSourceList?.find((dataSource) => dataSource.plugin_id === data.plugin_id)?.author
return matchDataSource(dataSourceList ?? [], data)?.declaration.identity.author
if (data.type === BlockEnum.Tool) {
if (data.provider_type === CollectionType.builtIn)
@@ -52,8 +54,10 @@ export const useNodeMetaData = (node: Node) => {
const description = useMemo(() => {
if (data.type === BlockEnum.DataSource)
return dataSourceList?.find((dataSource) => dataSource.plugin_id === data.plugin_id)
?.description[language]
return renderI18nObject(
matchDataSource(dataSourceList ?? [], data)?.declaration.identity.description,
language,
)
if (data.type === BlockEnum.Tool) {
if (data.provider_type === CollectionType.builtIn)
return buildInTools?.find((toolWithProvider) =>

View File

@@ -1,9 +1,11 @@
import type { RagPipelineDatasourceProviderResponse } from '@dify/contracts/api/console/rag/types.gen'
import type { TriggerWithProvider } from '../block-selector/types'
import type { DataSourceNodeType } from '../nodes/data-source/types'
import type { ToolNodeType } from '../nodes/tool/types'
import type { PluginTriggerNodeType } from '../nodes/trigger-plugin/types'
import type { Node, ToolWithProvider } from '../types'
import { useCallback, useMemo } from 'react'
import { resolveDatasourceIcon } from '@/app/components/rag-pipeline/utils/datasource-icon'
import { CollectionType } from '@/app/components/tools/types'
import useTheme from '@/hooks/use-theme'
import {
@@ -17,6 +19,7 @@ import { canFindTool } from '@/utils'
import { matchesProviderReference } from '@/utils/provider-reference'
import { useStore, useWorkflowStore } from '../store'
import { BlockEnum } from '../types'
import { matchDataSource } from '../utils/plugin-install-check'
const isTriggerPluginNode = (data: Node['data']): data is PluginTriggerNodeType =>
data.type === BlockEnum.TriggerPlugin
@@ -132,9 +135,12 @@ const findToolNodeIcon = ({
return resolveIconByTheme(theme, data.provider_icon, data.provider_icon_dark)
}
const findDataSourceIcon = (data: DataSourceNodeType, dataSourceList?: ToolWithProvider[]) => {
return dataSourceList?.find((toolWithProvider) => toolWithProvider.plugin_id === data.plugin_id)
?.icon
const findDataSourceIcon = (
data: DataSourceNodeType,
dataSourceList?: RagPipelineDatasourceProviderResponse[],
) => {
const provider = matchDataSource(dataSourceList ?? [], data)
return provider ? resolveDatasourceIcon(provider.declaration.identity.icon) : undefined
}
export const findNodeIcon = ({
@@ -146,7 +152,7 @@ export const findNodeIcon = ({
}: {
data?: Node['data']
collections: ToolIconCollections
dataSourceList?: ToolWithProvider[]
dataSourceList?: RagPipelineDatasourceProviderResponse[]
triggerPlugins?: TriggerWithProvider[]
theme?: string
}) => {

View File

@@ -7,6 +7,7 @@ import { fireEvent, screen } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { FormTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
import { renderWorkflowFlowComponent } from '@/app/components/workflow/__tests__/workflow-test-env'
import { VarType } from '@/app/components/workflow/types'
import { VarKindType } from '../../types'
import FormInputItem from '../form-input-item'
@@ -31,6 +32,7 @@ vi.mock('../../../../hooks/use-workflow-variables', async (importOriginal) => {
...actual,
useWorkflowVariables: () => ({
getNodeAvailableVars: () => [],
getCurrentVariableType: () => VarType.string,
}),
}
})
@@ -197,3 +199,24 @@ describe('FormInputItem', () => {
})
})
})
it.each([false, 0])(
'retains the static schema default %s when switching a variable to a constant',
(defaultValue) => {
const { onChange } = renderFormInputItem({
staticSchema: true,
schema: {
...createSchema({
type: typeof defaultValue === 'number' ? FormTypeEnum.textNumber : FormTypeEnum.checkbox,
_type: typeof defaultValue === 'number' ? FormTypeEnum.textNumber : FormTypeEnum.boolean,
}),
default: defaultValue,
},
value: { field: { type: VarKindType.variable, value: ['upstream', 'value'] } },
})
fireEvent.click(screen.getByRole('radio', { name: 'workflow.nodes.common.typeSwitch.input' }))
expect(onChange).toHaveBeenCalledWith({
field: { type: VarKindType.constant, value: defaultValue },
})
},
)

View File

@@ -11,8 +11,11 @@ import { FormTypeEnum } from '@/app/components/header/account-setting/model-prov
import { VarType } from '@/app/components/workflow/types'
import { VarKindType } from '../types'
type FormInputSchema = CredentialFormSchema &
export type FormInputSchema = Omit<CredentialFormSchema, 'default'> &
Partial<{
default: unknown
min: number
max: number
_type: FormTypeEnum
multiple: boolean
options: FormOption[]

View File

@@ -2,11 +2,8 @@
import type { FC } from 'react'
import type { ResourceVarInputs } from '../types'
import type {
CredentialFormSchema,
FormOption,
FormTypeEnum,
} from '@/app/components/header/account-setting/model-provider-page/declarations'
import type { FormInputSchema } from './form-input-item.helpers'
import type { FormOption } from '@/app/components/header/account-setting/model-provider-page/declarations'
import type { Event, Tool } from '@/app/components/tools/types'
import type { TriggerWithProvider } from '@/app/components/workflow/block-selector/types'
import type { ToolWithProvider, ValueSelector, Var } from '@/app/components/workflow/types'
@@ -57,7 +54,7 @@ type Props = Readonly<{
readOnly: boolean
labelId?: string
nodeId: string
schema: CredentialFormSchema
schema: FormInputSchema
value: ResourceVarInputs
onChange: (value: ResourceVarInputs) => void
inPanel?: boolean
@@ -67,6 +64,7 @@ type Props = Readonly<{
onManageInputField?: () => void
extraParams?: Record<string, unknown>
providerType?: 'tool' | 'trigger'
staticSchema?: boolean
disableVariableInsertion?: boolean
}>
@@ -92,6 +90,7 @@ const FormInputItem: FC<Props> = ({
onManageInputField,
extraParams,
providerType,
staticSchema = false,
disableVariableInsertion = false,
inPanel,
}) => {
@@ -104,15 +103,7 @@ const FormInputItem: FC<Props> = ({
const [toolsOptions, setToolsOptions] = useState<FormOption[] | null>(null)
const [isLoadingToolsOptions, setIsLoadingToolsOptions] = useState(false)
const formState = getFormInputState(
schema as CredentialFormSchema & {
_type?: FormTypeEnum
multiple?: boolean
options?: FormOption[]
scope?: string
},
value[schema.variable],
)
const formState = getFormInputState(schema, value[schema.variable])
const {
defaultValue,
@@ -168,7 +159,8 @@ const FormInputItem: FC<Props> = ({
extra: extraParams,
credential_id: currentProvider?.credential_id || '',
},
isDynamicSelect &&
!staticSchema &&
isDynamicSelect &&
providerType === PluginCategoryEnum.trigger &&
!!currentTool &&
!!currentProvider,
@@ -187,6 +179,7 @@ const FormInputItem: FC<Props> = ({
useEffect(() => {
const fetchPanelDynamicOptions = async () => {
if (
!staticSchema &&
isDynamicSelect &&
currentTool &&
currentProvider &&
@@ -207,6 +200,7 @@ const FormInputItem: FC<Props> = ({
fetchPanelDynamicOptions()
}, [
staticSchema,
isDynamicSelect,
currentTool?.name,
currentProvider?.name,
@@ -348,6 +342,8 @@ const FormInputItem: FC<Props> = ({
{isNumber && isConstant && (
<NumberField
step="any"
min={staticSchema ? schema.min : undefined}
max={staticSchema ? schema.max : undefined}
className="min-w-0 grow"
value={varInput?.value == null || varInput.value === '' ? null : Number(varInput.value)}
readOnly={readOnly}

View File

@@ -1,10 +1,12 @@
import type { AgentV2NodeType } from '@/app/components/workflow/nodes/agent-v2/types'
import type { AgentNodeType } from '@/app/components/workflow/nodes/agent/types'
import type { AnswerNodeType } from '@/app/components/workflow/nodes/answer/types'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import type { HumanInputNodeType } from '@/app/components/workflow/nodes/human-input/types'
import type { LLMNodeType } from '@/app/components/workflow/nodes/llm/types'
import type { EnvironmentVariable, Node, PromptItem } from '@/app/components/workflow/types'
import { describe, expect, it } from 'vite-plus/test'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { DeliveryMethodType } from '@/app/components/workflow/nodes/human-input/types'
import {
BlockEnum,
@@ -435,3 +437,60 @@ describe('variable utils', () => {
})
})
})
it('keeps datasource output variables typed through the public picker projection', () => {
const provider = createDatasourceProvider()
provider.declaration.datasources![0]!.output_schema = {
properties: {
title: { type: 'string' },
count: { type: 'integer' },
names: { type: 'array', items: { type: 'string' } },
metadata: {
type: 'object',
properties: { summary: { type: 'string' }, enabled: { type: 'boolean' } },
},
unrestricted: true,
forbidden: false,
unknown: null,
},
}
const node = createNode<DataSourceNodeType>({
type: BlockEnum.DataSource,
title: 'Datasource',
desc: '',
plugin_id: provider.plugin_id,
provider_name: provider.provider,
provider_type: 'local_file',
datasource_name: 'local-file',
datasource_label: 'Local file',
datasource_parameters: {},
datasource_configurations: {},
})
const options = {
beforeNodes: [node],
isChatMode: false,
allPluginInfoList: { dataSourceList: [provider] },
}
const all = toNodeAvailableVars({ ...options, filterVar: () => true }).find(
(item) => item.nodeId === node.id,
)?.vars
expect(all).toEqual(
expect.arrayContaining([
expect.objectContaining({ variable: 'file', type: VarType.file }),
expect.objectContaining({ variable: 'title', type: VarType.string }),
expect.objectContaining({ variable: 'count', type: VarType.integer }),
expect.objectContaining({ variable: 'names', type: VarType.arrayString }),
expect.objectContaining({ variable: 'unrestricted', type: VarType.any }),
expect.objectContaining({ variable: 'forbidden', type: VarType.any }),
expect.objectContaining({ variable: 'unknown', type: VarType.any }),
]),
)
const strings = toNodeAvailableVars({
...options,
filterVar: (variable) => variable.type === VarType.string,
}).find((item) => item.nodeId === node.id)?.vars
expect(strings?.find((variable) => variable.variable === 'metadata')?.children).toEqual([
expect.objectContaining({ variable: 'summary', type: VarType.string }),
])
expect(strings?.find((variable) => variable.variable === 'unrestricted')).toBeUndefined()
})

View File

@@ -0,0 +1,68 @@
import { fireEvent, render, screen } from '@testing-library/react'
import ShowPanel from '../index'
it('keeps known object fields, descriptions, enums, file markers and collapse behavior', () => {
const { container } = render(
<ShowPanel
payload={{
schema: {
properties: {
result: {
type: 'object',
required: ['title'],
description: 'Result details',
properties: {
title: { type: 'string', description: 'The title' },
names: { type: 'array', items: { type: 'string' } },
state: { type: 'string', enum: ['ready', 'done'] },
document: { type: 'object', schemaType: 'file' },
},
},
},
},
}}
/>,
)
expect(screen.getByText('The title')).toBeInTheDocument()
expect(screen.getByText('app.structOutput.required')).toBeInTheDocument()
expect(screen.getByText('array[string]')).toBeInTheDocument()
expect(screen.getByText('"ready" |')).toBeInTheDocument()
expect(screen.getByText('file')).toBeInTheDocument()
const collapse = container.querySelector('svg')
expect(collapse).not.toBeNull()
fireEvent.click(collapse!)
expect(screen.queryByText('title')).not.toBeInTheDocument()
expect(screen.getByText('Result details')).toBeInTheDocument()
fireEvent.click(collapse!)
expect(screen.getByText('title')).toBeInTheDocument()
})
it('renders unknown boolean, nullable and untyped nested schema fields without throwing', () => {
render(
<ShowPanel
payload={{
schema: {
properties: {
allowed: true,
blocked: false,
nullable: null,
untyped: {},
empty: { type: '' },
nested: { type: 'object', properties: { child: null } },
},
},
}}
/>,
)
expect(screen.getAllByText('Unknown')).toHaveLength(6)
for (const name of ['allowed', 'blocked', 'nullable', 'untyped', 'empty', 'child'])
expect(screen.getByText(name)).toBeInTheDocument()
})
it.each([null, true, [], { properties: null }, { properties: false }])(
'safely ignores a schema with no properties: %j',
(schema) => {
const { container } = render(<ShowPanel payload={{ schema }} />)
expect(container).toBeEmptyDOMElement()
},
)

View File

@@ -1,18 +1,16 @@
'use client'
import type { FC } from 'react'
import type { Field as FieldType } from '../../../../../llm/types'
import { cn } from '@langgenius/dify-ui/cn'
import { RiArrowDropDownLine } from '@remixicon/react'
import * as React from 'react'
import { useState } from 'react'
import { useTranslation } from 'react-i18next'
import { Type } from '../../../../../llm/types'
import { getFieldType } from '../../../../../llm/utils'
import TreeIndentLine from '../tree-indent-line'
type Props = Readonly<{
name: string
payload: FieldType
payload: unknown
required: boolean
depth?: number
rootClassName?: string
@@ -21,8 +19,47 @@ type Props = Readonly<{
const Field: FC<Props> = ({ name, payload, depth = 1, required, rootClassName }) => {
const { t } = useTranslation()
const isRoot = depth === 1
const hasChildren = payload.type === Type.object && payload.properties
const hasEnum = payload.enum && payload.enum.length > 0
const schema = payload && typeof payload === 'object' && !Array.isArray(payload) ? payload : {}
const type =
'type' in schema && typeof schema.type === 'string' && schema.type ? schema.type : 'Unknown'
const properties =
'properties' in schema &&
schema.properties &&
typeof schema.properties === 'object' &&
!Array.isArray(schema.properties)
? schema.properties
: undefined
const requiredProperties =
'required' in schema && Array.isArray(schema.required) ? schema.required : []
const description =
'description' in schema && typeof schema.description === 'string' ? schema.description : ''
const schemaType =
'schemaType' in schema && typeof schema.schemaType === 'string' ? schema.schemaType : ''
const enumValues =
'enum' in schema && Array.isArray(schema.enum)
? schema.enum.filter(
(value): value is string | number =>
typeof value === 'string' || typeof value === 'number',
)
: []
const items = 'items' in schema ? schema.items : undefined
const itemType =
items &&
typeof items === 'object' &&
'type' in items &&
typeof items.type === 'string' &&
items.type
? items.type
: 'Unknown'
const displayType =
schemaType === 'file'
? Type.file
: enumValues.length > 0
? Type.enumType
: type === Type.array && items
? `array[${itemType}]`
: type
const hasChildren = type === Type.object && properties
const [fold, setFold] = useState(false)
return (
<div>
@@ -48,8 +85,8 @@ const Field: FC<Props> = ({ name, payload, depth = 1, required, rootClassName })
{name}
</div>
<div className="ml-3 shrink-0 system-xs-regular leading-6 text-text-tertiary">
{getFieldType(payload)}
{payload.schemaType && payload.schemaType !== 'file' && ` (${payload.schemaType})`}
{displayType}
{schemaType && schemaType !== 'file' && ` (${schemaType})`}
</div>
{required && (
<div className="ml-3 system-2xs-medium-uppercase leading-6 text-text-warning">
@@ -57,20 +94,20 @@ const Field: FC<Props> = ({ name, payload, depth = 1, required, rootClassName })
</div>
)}
</div>
{payload.description && (
{description && (
<div className="ml-1.75 flex">
<div className="w-0 grow truncate system-xs-regular text-text-tertiary">
{payload.description}
{description}
</div>
</div>
)}
{hasEnum && (
{enumValues.length > 0 && (
<div className="ml-1.75 flex">
<div className="w-0 grow system-xs-regular text-text-quaternary">
{payload.enum!.map((value, index) => (
{enumValues.map((value, index) => (
<span key={index}>
{typeof value === 'string' ? `"${value}"` : value}
{index < payload.enum!.length - 1 && ' | '}
{index < enumValues.length - 1 && ' | '}
</span>
))}
</div>
@@ -81,13 +118,13 @@ const Field: FC<Props> = ({ name, payload, depth = 1, required, rootClassName })
{hasChildren && !fold && (
<div>
{Object.keys(payload.properties!).map((name) => (
{Object.entries(properties).map(([name, value]: [string, unknown]) => (
<Field
key={name}
name={name}
payload={payload.properties?.[name] as FieldType}
payload={value}
depth={depth + 1}
required={!!payload.required?.includes(name)}
required={requiredProperties.includes(name)}
/>
))}
</div>

View File

@@ -1,32 +1,28 @@
'use client'
import type { FC } from 'react'
import type { StructuredOutput } from '../../../../../llm/types'
import * as React from 'react'
import { useTranslation } from 'react-i18next'
import Field from './field'
type Props = Readonly<{
payload: StructuredOutput
payload: { schema: unknown }
rootClassName?: string
}>
const ShowPanel: FC<Props> = ({ payload, rootClassName }) => {
const { t } = useTranslation()
const schema = {
...payload,
schema: {
...payload.schema,
description: t(($) => $['structOutput.LLMResponse'], { ns: 'app' }),
},
}
const schema = payload.schema
if (!schema || typeof schema !== 'object' || !('properties' in schema)) return null
const properties = schema.properties
if (!properties || typeof properties !== 'object' || Array.isArray(properties)) return null
const required = 'required' in schema && Array.isArray(schema.required) ? schema.required : []
return (
<div className="relative -left-1.75">
{Object.keys(schema.schema.properties!).map((name) => (
{Object.entries(properties).map(([name, field]: [string, unknown]) => (
<Field
key={name}
name={name}
payload={schema.schema.properties![name]!}
required={!!schema.schema.required?.includes(name)}
payload={field}
required={required.includes(name)}
rootClassName={rootClassName}
/>
))}

View File

@@ -30,9 +30,9 @@ import type {
Node,
NodeOutPutVar,
PromptItem,
ToolWithProvider,
ValueSelector,
Var,
WorkflowPluginCatalogs,
} from '@/app/components/workflow/types'
import type { RAGPipelineVariable } from '@/models/pipeline'
import type { SchemaTypeDefinition } from '@/service/use-common'
@@ -316,7 +316,7 @@ const formatItem = (
item: any,
isChatMode: boolean,
filterVar: (payload: Var, selector: ValueSelector) => boolean,
allPluginInfoList: Record<string, ToolWithProvider[]>,
allPluginInfoList: WorkflowPluginCatalogs,
ragVars?: Var[],
schemaTypeDefinitions: SchemaTypeDefinition[] = [],
): NodeOutPutVar => {
@@ -761,7 +761,7 @@ export const toNodeOutputVars = (
environmentVariables: EnvironmentVariable[] = [],
conversationVariables: ConversationVariable[] = [],
ragVariables: RAGPipelineVariable[] = [],
allPluginInfoList: Record<string, ToolWithProvider[]>,
allPluginInfoList: WorkflowPluginCatalogs,
schemaTypeDefinitions?: SchemaTypeDefinition[],
): NodeOutPutVar[] => {
// ENV_NODE data format
@@ -980,7 +980,7 @@ export const getVarType = ({
environmentVariables?: EnvironmentVariable[]
conversationVariables?: ConversationVariable[]
ragVariables?: RAGPipelineVariable[]
allPluginInfoList: Record<string, ToolWithProvider[]>
allPluginInfoList: WorkflowPluginCatalogs
schemaTypeDefinitions?: SchemaTypeDefinition[]
preferSchemaType?: boolean
}): VarType => {
@@ -1124,7 +1124,7 @@ export const toNodeAvailableVars = ({
// rag variables
ragVariables?: RAGPipelineVariable[]
filterVar: (payload: Var, selector: ValueSelector) => boolean
allPluginInfoList: Record<string, ToolWithProvider[]>
allPluginInfoList: WorkflowPluginCatalogs
schemaTypeDefinitions?: SchemaTypeDefinition[]
}): NodeOutPutVar[] => {
const beforeNodesOutputVars = toNodeOutputVars(

View File

@@ -1,9 +1,7 @@
'use client'
import type {
CredentialFormSchema,
FormOption,
} from '@/app/components/header/account-setting/model-provider-page/declarations'
import type { FormInputSchema } from '../form-input-item.helpers'
import type { FormOption } from '@/app/components/header/account-setting/model-provider-page/declarations'
import type { VarKindType } from '@/app/components/workflow/nodes/_base/types'
import type {
CommonNodeType,
@@ -24,7 +22,7 @@ import {
type DynamicSchemaParams = {
dynamicOptions: FormOption[] | null
isLoading: boolean
schema?: Partial<CredentialFormSchema>
schema?: Partial<Omit<FormInputSchema, 'default'>>
value: ValueSelector | string
}

View File

@@ -1,10 +1,8 @@
'use client'
import type { FC, ReactElement } from 'react'
import type {
CredentialFormSchema,
CredentialFormSchemaSelect,
} from '@/app/components/header/account-setting/model-provider-page/declarations'
import type { FormInputSchema } from '../form-input-item.helpers'
import type { CredentialFormSchemaSelect } from '@/app/components/header/account-setting/model-provider-page/declarations'
import type { Tool } from '@/app/components/tools/types'
import type { TriggerWithProvider } from '@/app/components/workflow/block-selector/types'
import type { VarKindType } from '@/app/components/workflow/nodes/_base/types'
@@ -71,7 +69,7 @@ type Props = Readonly<{
outputVarNodeId?: string
placeholder?: string
readonly: boolean
schemaWithDynamicSelect?: Partial<CredentialFormSchema>
schemaWithDynamicSelect?: Partial<Omit<FormInputSchema, 'default'>>
setControlFocus: (value: number) => void
setOpen: (value: boolean) => void
showErrorIcon?: boolean

View File

@@ -1,8 +1,9 @@
'use client'
import type { FC } from 'react'
import type { FormInputSchema } from '../form-input-item.helpers'
import type { HoverPopup } from './var-reference-picker.trigger'
import type {
CredentialFormSchema,
CredentialFormSchemaSelect,
FormOption,
} from '@/app/components/header/account-setting/model-provider-page/declarations'
@@ -74,7 +75,7 @@ type Props = Readonly<{
isAddBtnTrigger?: boolean
trigger?: React.ReactNode
isJustShowValue?: boolean
schema?: Partial<CredentialFormSchema>
schema?: Partial<Omit<FormInputSchema, 'default'>>
valueTypePlaceHolder?: string
isInTable?: boolean
onRemove?: () => void

View File

@@ -1,6 +1,7 @@
import type { TriggerWithProvider } from '@/app/components/workflow/block-selector/types'
import type { CustomRunFormProps } from '@/app/components/workflow/nodes/data-source/types'
import type { Node, ToolWithProvider } from '@/app/components/workflow/types'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { BlockEnum } from '@/app/components/workflow/types'
import {
clampNodePanelWidth,
@@ -76,7 +77,7 @@ describe('workflow-panel helpers', () => {
provider_type: 'online_document',
})
const triggerPlugins = [{ plugin_id: 'trigger-1', id: '1' }]
const dataSources = [{ plugin_id: 'source-1' }]
const dataSources = [createDatasourceProvider({ plugin_id: 'source-1' })]
expect(getCurrentTriggerPlugin(triggerData, asTriggerList(triggerPlugins))).toEqual(
triggerPlugins[0],

View File

@@ -1,8 +1,8 @@
import type { PropsWithChildren } from 'react'
import type { ToolWithProvider } from '@/app/components/workflow/types'
import { act, fireEvent, screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import * as React from 'react'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { renderWorkflowComponent } from '@/app/components/workflow/__tests__/workflow-test-env'
import { BlockEnum, NodeRunningStatus } from '@/app/components/workflow/types'
import BasePanel from '../index'
@@ -66,25 +66,6 @@ const mockLastRunState = {
getFilteredExistVarForms: vi.fn(() => []),
}
const createDataSourceCollection = (overrides: Partial<ToolWithProvider> = {}): ToolWithProvider =>
({
id: 'source-1',
name: 'Source',
author: 'Author',
description: { en_US: 'Source description', zh_Hans: 'Source description' },
icon: 'source-icon',
label: { en_US: 'Source', zh_Hans: 'Source' },
type: 'datasource',
team_credentials: {},
is_team_authorization: false,
allow_delete: false,
labels: [],
plugin_id: 'source-1',
tools: [],
meta: {} as ToolWithProvider['meta'],
...overrides,
}) as ToolWithProvider
vi.mock('@/app/components/app/store', () => ({
useStore: (
selector: (state: { showMessageLogModal: boolean; appDetail: { id: string } }) => unknown,
@@ -665,7 +646,9 @@ describe('workflow-panel index', () => {
initialStoreState: {
nodePanelWidth: 480,
otherPanelWidth: 200,
dataSourceList: [createDataSourceCollection({ is_authorized: false })],
dataSourceList: [
createDatasourceProvider({ plugin_id: 'source-1', is_authorized: false }),
],
},
},
)

View File

@@ -1,3 +1,4 @@
import type { RagPipelineDatasourceProviderResponse } from '@dify/contracts/api/console/rag/types.gen'
import type { ReactNode } from 'react'
import type { TriggerWithProvider } from '@/app/components/workflow/block-selector/types'
import type { CustomRunFormProps } from '@/app/components/workflow/nodes/data-source/types'
@@ -63,7 +64,7 @@ export const getCurrentToolCollection = (
export const getCurrentDataSource = (
data: Node['data'],
dataSourceList: Array<{ plugin_id?: string; is_authorized?: boolean }> | undefined,
dataSourceList: RagPipelineDatasourceProviderResponse[] | undefined,
) => {
if (
data.type !== BlockEnum.DataSource ||

View File

@@ -408,14 +408,16 @@ const BasePanel: FC<BasePanelProps> = ({ id, data, children }) => {
)
const readmeEntranceComponent = useMemo(() => {
if (data.type === BlockEnum.DataSource)
return currentDataSource ? (
<ReadmeEntrance pluginDetail={currentDataSource} className="mt-auto" />
) : null
let pluginDetail
switch (data.type) {
case BlockEnum.Tool:
pluginDetail = currToolCollection
break
case BlockEnum.DataSource:
pluginDetail = currentDataSource
break
case BlockEnum.TriggerPlugin:
pluginDetail = currentTriggerPlugin
break

View File

@@ -1,222 +1,158 @@
import type { ReactNode } from 'react'
import type { DataSourceNodeType } from '../types'
import type { NodePanelProps } from '@/app/components/workflow/types'
import { fireEvent, render, screen } from '@testing-library/react'
import { toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema'
import { useStore } from '@/app/components/workflow/store'
import { BlockEnum, VarType } from '@/app/components/workflow/types'
import { useNodesReadOnly } from '../../../hooks/use-workflow'
import useMatchSchemaType, {
getMatchedSchemaType,
} from '../../_base/components/variable/use-match-schema-type'
import ToolForm from '../../tool/components/tool-form'
import { useConfig } from '../hooks/use-config'
import { screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { renderWorkflowFlowComponent } from '@/app/components/workflow/__tests__/workflow-test-env'
import { VarKindType } from '@/app/components/workflow/nodes/_base/types'
import { BlockEnum } from '@/app/components/workflow/types'
import Panel from '../panel'
const mockWrapStructuredVarItem = vi.hoisted(() => vi.fn((payload: unknown) => payload))
vi.mock('@/app/components/base/tag-input', () => ({
__esModule: true,
default: ({
items,
onChange,
placeholder,
}: {
items: string[]
onChange: (items: string[]) => void
placeholder?: string
}) => (
<button type="button" onClick={() => onChange([...items, 'txt'])}>
{placeholder}
</button>
),
const { update, get } = vi.hoisted(() => ({
update: vi.fn(),
get: vi.fn(async (_url: string) => []),
}))
vi.mock('@/service/base', async (importOriginal) => ({
...(await importOriginal<typeof import('@/service/base')>()),
get,
}))
vi.mock('../../../hooks/use-node-data-update', () => ({
useNodeDataUpdate: () => ({ handleNodeDataUpdateWithSyncDraft: update }),
}))
vi.mock('../../../hooks/use-workflow', async (importOriginal) => ({
...(await importOriginal<typeof import('../../../hooks/use-workflow')>()),
useNodesReadOnly: () => ({ nodesReadOnly: false }),
}))
vi.mock('../../_base/hooks/use-available-var-list', () => ({
default: () => ({ availableVars: [], availableNodesWithParent: [] }),
}))
vi.mock('@/app/components/tools/utils/to-form-schema', () => ({
toolParametersToFormSchemas: vi.fn(),
}))
vi.mock('../../../hooks/use-workflow', async (importOriginal) => {
const actual = await importOriginal<typeof import('../../../hooks/use-workflow')>()
return {
...actual,
useNodesReadOnly: vi.fn(),
}
})
vi.mock('@/app/components/workflow/store', () => ({
useStore: vi.fn(),
}))
vi.mock('@/app/components/workflow/utils/tool', () => ({
wrapStructuredVarItem: (payload: unknown) => mockWrapStructuredVarItem(payload),
}))
vi.mock('../../_base/components/output-vars', () => ({
__esModule: true,
default: ({ children }: { children: ReactNode }) => <div>{children}</div>,
VarItem: ({ name, type }: { name: string; type: string }) => <div>{`${name}:${type}`}</div>,
}))
vi.mock('../../_base/components/variable/object-child-tree-panel/show', () => ({
__esModule: true,
default: ({ payload }: { payload: { name: string } }) => <div>{payload.name}</div>,
}))
vi.mock('../../_base/components/variable/use-match-schema-type', () => ({
__esModule: true,
default: vi.fn(),
getMatchedSchemaType: vi.fn(),
}))
vi.mock('../../tool/components/tool-form', () => ({
__esModule: true,
default: vi.fn(
({
onChange,
onManageInputField,
}: {
onChange: (value: unknown) => void
onManageInputField?: () => void
}) => (
<div>
<button type="button" onClick={() => onChange({ dataset: 'docs' })}>
tool-form-change
</button>
<button type="button" onClick={() => onManageInputField?.()}>
manage-input-field
</button>
</div>
),
),
}))
vi.mock('../hooks/use-config', () => ({
useConfig: vi.fn(),
}))
const mockUseNodesReadOnly = vi.mocked(useNodesReadOnly)
const mockUseStore = vi.mocked(useStore)
const mockUseConfig = vi.mocked(useConfig)
const mockToolParametersToFormSchemas = vi.mocked(toolParametersToFormSchemas)
const mockUseMatchSchemaType = vi.mocked(useMatchSchemaType)
const mockGetMatchedSchemaType = vi.mocked(getMatchedSchemaType)
const mockToolForm = vi.mocked(ToolForm)
const setShowInputFieldPanel = vi.fn()
const panelProps = {
getInputVars: () => [],
toVarInputs: () => [],
runInputData: {},
runInputDataRef: { current: {} },
setRunInputData: () => {},
runResult: null,
}
const createData = (overrides: Partial<DataSourceNodeType> = {}): DataSourceNodeType => ({
title: 'Datasource',
desc: '',
type: BlockEnum.DataSource,
plugin_id: 'plugin-1',
plugin_id: 'langgenius/file',
provider_type: 'online_document',
provider_name: 'provider',
datasource_name: 'source-a',
datasource_label: 'Source A',
provider_name: 'file',
datasource_name: 'local-file',
datasource_label: 'File',
datasource_parameters: {},
datasource_configurations: {},
fileExtensions: ['pdf'],
...overrides,
})
const createProvider = () => {
const provider = createDatasourceProvider()
provider.declaration.datasources![0]!.parameters = [
{
name: 'count',
type: 'number',
required: true,
default: 0,
min: 0,
max: 10,
label: { en_US: 'Count', zh_Hans: null },
description: { en_US: 'How many documents' },
},
{
name: 'enabled',
type: 'boolean',
required: false,
default: false,
label: { en_US: 'Enabled' },
description: { en_US: 'Include archived documents' },
},
]
return provider
}
beforeEach(() => vi.clearAllMocks())
const panelProps = {} as NodePanelProps<DataSourceNodeType>['panelProps']
describe('data-source/panel', () => {
beforeEach(() => {
vi.clearAllMocks()
mockUseNodesReadOnly.mockReturnValue({ nodesReadOnly: false, getNodesReadOnly: () => false })
mockUseStore.mockImplementation((selector) => {
const select = selector as (state: unknown) => unknown
return select({
dataSourceList: [
{
plugin_id: 'plugin-1',
is_authorized: true,
tools: [
{
name: 'source-a',
parameters: [{ name: 'dataset' }],
},
],
},
],
pipelineId: 'pipeline-1',
setShowInputFieldPanel,
})
})
mockUseConfig.mockReturnValue({
handleFileExtensionsChange: vi.fn(),
handleParametersChange: vi.fn(),
outputSchema: [],
hasObjectOutput: false,
})
mockToolParametersToFormSchemas.mockReturnValue([{ name: 'dataset' }] as never)
mockUseMatchSchemaType.mockReturnValue({ schemaTypeDefinitions: {} } as ReturnType<
typeof useMatchSchemaType
>)
mockGetMatchedSchemaType.mockReturnValue('')
it('renders real datasource fields and descriptions without tool dynamic requests, preserving numeric and boolean values', async () => {
const user = userEvent.setup()
const data = createData({
datasource_parameters: {
count: { type: VarKindType.constant, value: 0 },
enabled: { type: VarKindType.constant, value: false },
},
})
it('renders the authorized tool form path and forwards parameter changes', () => {
const handleParametersChange = vi.fn()
mockUseConfig.mockReturnValueOnce({
handleFileExtensionsChange: vi.fn(),
handleParametersChange,
outputSchema: [
{
name: 'metadata',
value: { type: 'object' },
},
],
hasObjectOutput: true,
})
mockGetMatchedSchemaType.mockReturnValueOnce('json')
render(<Panel id="data-source-node" data={createData()} panelProps={panelProps} />)
fireEvent.click(screen.getByRole('button', { name: 'tool-form-change' }))
fireEvent.click(screen.getByRole('button', { name: 'manage-input-field' }))
expect(handleParametersChange).toHaveBeenCalledWith({ dataset: 'docs' })
expect(setShowInputFieldPanel).toHaveBeenCalledWith(true)
expect(mockToolForm).toHaveBeenCalledWith(
expect.objectContaining({
nodeId: 'data-source-node',
showManageInputField: true,
value: {},
renderWorkflowFlowComponent(<Panel id="datasource" data={data} panelProps={panelProps} />, {
nodes: [{ id: 'datasource', position: { x: 0, y: 0 }, data }],
initialStoreState: { dataSourceList: [createProvider()] },
})
expect(screen.getByText('How many documents')).toBeInTheDocument()
const count = screen.getByRole('textbox', { name: 'Count' })
expect(count).toHaveValue('0')
await user.click(count)
await user.keyboard('{ArrowDown}')
expect(count).toHaveValue('0')
await user.clear(count)
await user.type(count, '2')
expect(update).toHaveBeenLastCalledWith({
id: 'datasource',
data: expect.objectContaining({
datasource_parameters: expect.objectContaining({
count: { type: VarKindType.constant, value: 2 },
}),
undefined,
)
expect(screen.getByText('metadata')).toBeInTheDocument()
}),
})
it('renders the local-file path and updates supported file extensions', () => {
const handleFileExtensionsChange = vi.fn()
mockUseConfig.mockReturnValueOnce({
handleFileExtensionsChange,
handleParametersChange: vi.fn(),
outputSchema: [],
hasObjectOutput: false,
})
render(
<Panel
id="data-source-node"
data={createData({ provider_type: 'local_file' })}
panelProps={panelProps}
/>,
)
fireEvent.click(
screen.getByRole('button', {
name: 'workflow.nodes.dataSource.supportedFileFormatsPlaceholder',
await user.click(screen.getByRole('button', { name: 'False' }))
expect(update).toHaveBeenLastCalledWith({
id: 'datasource',
data: expect.objectContaining({
datasource_parameters: expect.objectContaining({
enabled: { type: VarKindType.constant, value: false },
}),
)
expect(handleFileExtensionsChange).toHaveBeenCalledWith(['pdf', 'txt'])
expect(screen.getByText(`datasource_type:${VarType.string}`)).toBeInTheDocument()
expect(screen.getByText(`file:${VarType.file}`)).toBeInTheDocument()
}),
})
expect(get.mock.calls.map((call) => call[0])).not.toContain(
'/workspaces/current/plugin/parameters/dynamic-options',
)
})
it('renders nested and unrestricted outputs while keeping unauthorized inputs hidden', async () => {
const user = userEvent.setup()
const provider = createProvider()
provider.is_authorized = false
provider.declaration.datasources![0]!.output_schema = {
properties: {
metadata: {
type: 'object',
properties: { title: { type: 'string' }, optional: true, nullable: null },
},
names: { type: 'array', items: { type: 'string' } },
unknown: false,
},
}
const data = createData()
renderWorkflowFlowComponent(<Panel id="datasource" data={data} panelProps={panelProps} />, {
nodes: [{ id: 'datasource', position: { x: 0, y: 0 }, data }],
initialStoreState: { dataSourceList: [provider] },
})
await user.click(screen.getByRole('button', { name: 'workflow.nodes.common.outputVars' }))
await waitFor(() => expect(screen.getByText('metadata')).toBeInTheDocument())
expect(screen.getByText('title')).toBeInTheDocument()
expect(screen.getByText('optional')).toBeInTheDocument()
expect(screen.getByText('nullable')).toBeInTheDocument()
expect(screen.getByText('array[string]')).toBeInTheDocument()
expect(screen.queryByRole('textbox', { name: 'Count' })).not.toBeInTheDocument()
})
it('keeps local-file controls available without plugin version metadata', async () => {
const user = userEvent.setup()
const data = createData({ provider_type: 'local_file', fileExtensions: ['pdf', 'csv'] })
renderWorkflowFlowComponent(<Panel id="datasource" data={data} panelProps={panelProps} />, {
nodes: [{ id: 'datasource', position: { x: 0, y: 0 }, data }],
initialStoreState: { dataSourceList: [createDatasourceProvider()] },
})
expect(screen.getByText('pdf')).toBeInTheDocument()
expect(screen.getByText('csv')).toBeInTheDocument()
await user.click(screen.getByRole('button', { name: 'workflow.nodes.common.outputVars' }))
expect(screen.getByText('datasource_type')).toBeInTheDocument()
expect(screen.getByText('transfer_method')).toBeInTheDocument()
})

View File

@@ -1,10 +1,13 @@
import type { TFunction } from 'i18next'
import type { NodeDefault } from '../../types'
import type { NodeDefault, Var } from '../../types'
import type { getDataSourceCheckParams } from '../../utils/data-source'
import type { DataSourceNodeType } from './types'
import type { SchemaTypeDefinition } from '@/service/use-common'
import { VarKindType } from '@/app/components/workflow/nodes/_base/types'
import { BlockEnum } from '@/app/components/workflow/types'
import { genNodeMetaData } from '@/app/components/workflow/utils'
import { getMatchedSchemaType } from '../_base/components/variable/use-match-schema-type'
import { matchDataSource } from '../../utils/plugin-install-check'
import { resolveVarType } from '../tool/output-schema-utils'
import { COMMON_OUTPUT, LOCAL_FILE_OUTPUT } from './constants'
import { DataSourceClassification } from './types'
@@ -16,23 +19,54 @@ const metaData = genNodeMetaData({
isStart: true,
isRequired: true,
})
const getOutputProperties = (
schema: unknown,
schemaTypeDefinitions?: SchemaTypeDefinition[],
): Var[] => {
if (!schema || typeof schema !== 'object' || !('properties' in schema)) return []
const properties = schema.properties
if (!properties || typeof properties !== 'object' || Array.isArray(properties)) return []
return Object.entries(properties).map(([variable, value]: [string, unknown]) => {
const schemaValue =
value && typeof value === 'object' && !Array.isArray(value) ? value : undefined
const { type, schemaType } = resolveVarType(schemaValue, schemaTypeDefinitions)
return {
variable,
type,
schemaType,
des:
value &&
typeof value === 'object' &&
'description' in value &&
typeof value.description === 'string'
? value.description
: undefined,
children: type === 'object' ? getOutputProperties(value, schemaTypeDefinitions) : undefined,
}
})
}
const nodeDefault: NodeDefault<DataSourceNodeType> = {
metaData,
defaultValue: {
datasource_parameters: {},
datasource_configurations: {},
},
checkValid(payload, t: TFunction<'workflow'>, moreDataForCheckValid) {
checkValid(
payload,
t: TFunction<'workflow'>,
moreDataForCheckValid: ReturnType<typeof getDataSourceCheckParams>,
) {
const { dataSourceInputsSchema, notAuthed } = moreDataForCheckValid
let errorMessage = ''
if (notAuthed) errorMessage = t(($) => $[`${i18nPrefix}.authRequired`], { ns: 'workflow' })
if (!errorMessage) {
dataSourceInputsSchema
.filter((field: any) => {
.filter((field) => {
return field.required
})
.forEach((field: any) => {
.forEach((field) => {
const targetVar = payload.datasource_parameters[field.variable]
if (!targetVar) {
errorMessage = t(($) => $[`${i18nPrefix}.fieldRequired`], {
@@ -69,47 +103,16 @@ const nodeDefault: NodeDefault<DataSourceNodeType> = {
ragVars = [],
{ schemaTypeDefinitions } = { schemaTypeDefinitions: [] },
) {
const { plugin_id, datasource_name, provider_type } = payload
const { datasource_name, provider_type } = payload
const isLocalFile = provider_type === DataSourceClassification.localFile
const currentDataSource = allPluginInfoList.dataSourceList?.find(
(ds: any) => ds.plugin_id === plugin_id,
const provider = matchDataSource(allPluginInfoList.dataSourceList ?? [], payload)
const datasource = provider?.declaration.datasources?.find(
(item) => item.identity.name === datasource_name,
)
const currentDataSourceItem = currentDataSource?.tools?.find(
(tool: any) => tool.name === datasource_name,
const dynamicOutputSchema = getOutputProperties(
datasource?.output_schema,
schemaTypeDefinitions,
)
const output_schema = currentDataSourceItem?.output_schema
const dynamicOutputSchema: any[] = []
if (output_schema?.properties) {
Object.keys(output_schema.properties).forEach((outputKey) => {
const output = output_schema.properties[outputKey]
const dataType = output.type
let type =
dataType === 'array'
? `array[${output.items?.type.slice(0, 1).toLocaleLowerCase()}${output.items?.type.slice(1)}]`
: `${dataType.slice(0, 1).toLocaleLowerCase()}${dataType.slice(1)}`
const schemaType = getMatchedSchemaType?.(output, schemaTypeDefinitions)
if (type === 'object' && schemaType === 'file') type = 'file'
dynamicOutputSchema.push({
variable: outputKey,
type,
description: output.description,
schemaType,
children:
output.type === 'object'
? {
schema: {
type: 'object',
properties: output.properties,
},
}
: undefined,
})
})
}
return [
...COMMON_OUTPUT.map((item) => ({ variable: item.name, type: item.type })),
...(isLocalFile

View File

@@ -1,150 +1,84 @@
import type { DataSourceNodeType } from '../../types'
import { renderHook } from '@testing-library/react'
import { act } from '@testing-library/react'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { renderWorkflowFlowHook } from '@/app/components/workflow/__tests__/workflow-test-env'
import { VarKindType } from '@/app/components/workflow/nodes/_base/types'
import { BlockEnum } from '@/app/components/workflow/types'
import { useConfig } from '../use-config'
const mockUseStoreApi = vi.hoisted(() => vi.fn())
const mockUseNodeDataUpdate = vi.hoisted(() => vi.fn())
vi.mock('reactflow', () => ({
useStoreApi: () => mockUseStoreApi(),
const { update } = vi.hoisted(() => ({ update: vi.fn() }))
vi.mock('../../../../hooks/use-node-data-update', () => ({
useNodeDataUpdate: () => ({ handleNodeDataUpdateWithSyncDraft: update }),
}))
const data: DataSourceNodeType = {
title: 'Datasource',
desc: '',
type: BlockEnum.DataSource,
plugin_id: 'langgenius/file',
provider_type: 'local_file',
provider_name: 'file',
datasource_name: 'local-file',
datasource_label: 'File',
datasource_parameters: {},
datasource_configurations: {},
_dataSourceStartToAdd: true,
}
const nodes = [{ id: 'datasource', position: { x: 0, y: 0 }, data }]
beforeEach(() => vi.clearAllMocks())
vi.mock('../../../../hooks/use-node-data-update', async (importOriginal) => {
const actual = await importOriginal<typeof import('../../../../hooks/use-node-data-update')>()
it('clears local-file initialization and saves parameters and extensions through the draft owner', () => {
const { result } = renderWorkflowFlowHook(() => useConfig('datasource', data), { nodes })
expect(update).toHaveBeenCalledWith({
id: 'datasource',
data: { ...data, _dataSourceStartToAdd: false },
})
act(() => result.current.handleFileExtensionsChange(['pdf', 'csv']))
expect(update).toHaveBeenLastCalledWith({
id: 'datasource',
data: { ...data, fileExtensions: ['pdf', 'csv'] },
})
act(() =>
result.current.handleParametersChange({ count: { type: VarKindType.constant, value: 0 } }),
)
expect(update).toHaveBeenLastCalledWith({
id: 'datasource',
data: { ...data, datasource_parameters: { count: { type: VarKindType.constant, value: 0 } } },
})
})
return {
...actual,
useNodeDataUpdate: () => mockUseNodeDataUpdate(),
it('reads generated datasource output declarations and safely keeps unknown property schemas visible', () => {
const provider = createDatasourceProvider()
provider.declaration.datasources![0]!.output_schema = {
properties: {
metadata: { type: 'object', properties: { title: { type: 'string' } } },
names: { type: 'array', items: { type: 'string' }, description: 'Names' },
missingItems: { type: 'array' },
unrestricted: true,
forbidden: false,
empty: {},
nullable: null,
},
}
const { result } = renderWorkflowFlowHook(() => useConfig('datasource', data, [provider]), {
nodes,
})
expect(result.current.hasObjectOutput).toBe(true)
expect(result.current.outputSchema.map(({ name, type }) => ({ name, type }))).toEqual([
{ name: 'metadata', type: 'object' },
{ name: 'names', type: 'array[string]' },
{ name: 'missingItems', type: 'array[unknown]' },
{ name: 'unrestricted', type: 'unknown' },
{ name: 'forbidden', type: 'unknown' },
{ name: 'empty', type: 'unknown' },
{ name: 'nullable', type: 'unknown' },
])
})
const createNode = (
overrides: Partial<DataSourceNodeType> = {},
): { id: string; data: DataSourceNodeType } => ({
id: 'data-source-node',
data: {
title: 'Datasource',
desc: '',
type: 'data-source',
plugin_id: 'plugin-1',
provider_type: 'local_file',
provider_name: 'provider',
datasource_name: 'source-a',
datasource_label: 'Source A',
datasource_parameters: {},
datasource_configurations: {},
_dataSourceStartToAdd: true,
...overrides,
} as DataSourceNodeType,
})
describe('data-source/hooks/use-config', () => {
const mockHandleNodeDataUpdateWithSyncDraft = vi.fn()
let currentNode = createNode()
beforeEach(() => {
vi.clearAllMocks()
currentNode = createNode()
mockUseStoreApi.mockReturnValue({
getState: () => ({
getNodes: () => [currentNode],
}),
})
mockUseNodeDataUpdate.mockReturnValue({
handleNodeDataUpdateWithSyncDraft: mockHandleNodeDataUpdateWithSyncDraft,
})
})
it('should clear the local-file auto-add flag on mount and update datasource payloads', () => {
const { result } = renderHook(() => useConfig('data-source-node'))
expect(mockHandleNodeDataUpdateWithSyncDraft).toHaveBeenCalledWith({
id: 'data-source-node',
data: expect.objectContaining({
_dataSourceStartToAdd: false,
}),
})
mockHandleNodeDataUpdateWithSyncDraft.mockClear()
result.current.handleFileExtensionsChange(['pdf', 'csv'])
result.current.handleParametersChange({
dataset: {
type: VarKindType.constant,
value: 'docs',
},
})
expect(mockHandleNodeDataUpdateWithSyncDraft).toHaveBeenNthCalledWith(1, {
id: 'data-source-node',
data: expect.objectContaining({
fileExtensions: ['pdf', 'csv'],
}),
})
expect(mockHandleNodeDataUpdateWithSyncDraft).toHaveBeenNthCalledWith(2, {
id: 'data-source-node',
data: expect.objectContaining({
datasource_parameters: {
dataset: {
type: VarKindType.constant,
value: 'docs',
},
},
}),
})
})
it('should derive output schema metadata and detect object outputs', () => {
const dataSourceList = [
{
plugin_id: 'plugin-1',
tools: [
{
name: 'source-a',
output_schema: {
properties: {
items: {
type: 'array',
items: { type: 'string' },
description: 'List of items',
},
metadata: {
type: 'object',
description: 'Object field',
},
count: {
type: 'number',
description: 'Total count',
},
},
},
},
],
},
]
const { result } = renderHook(() => useConfig('data-source-node', dataSourceList))
expect(result.current.outputSchema).toEqual([
{
name: 'items',
type: 'Array[String]',
description: 'List of items',
},
{
name: 'metadata',
value: {
type: 'object',
description: 'Object field',
},
},
{
name: 'count',
type: 'Number',
description: 'Total count',
},
])
expect(result.current.hasObjectOutput).toBe(true)
})
it('accepts null output schemas and absent catalog data', () => {
const { result } = renderWorkflowFlowHook(
() => useConfig('datasource', data, [createDatasourceProvider()]),
{ nodes },
)
expect(result.current.outputSchema).toEqual([])
expect(result.current.hasObjectOutput).toBe(false)
})

View File

@@ -1,18 +1,24 @@
import type { DataSourceNodeType, ToolVarInputs } from '../types'
import type { RagPipelineDatasourceProviderResponse } from '@dify/contracts/api/console/rag/types.gen'
import type { ResourceVarInputs } from '../../_base/types'
import type { DataSourceNodeType } from '../types'
import { useCallback, useEffect, useMemo } from 'react'
import { useStoreApi } from 'reactflow'
import { useReactFlow } from 'reactflow'
import { matchDataSource } from '@/app/components/workflow/utils/plugin-install-check'
import { useNodeDataUpdate } from '../../../hooks/use-node-data-update'
export const useConfig = (id: string, dataSourceList?: any[]) => {
const store = useStoreApi()
export const useConfig = (
id: string,
data: DataSourceNodeType,
dataSourceList?: RagPipelineDatasourceProviderResponse[],
) => {
const { getNodes } = useReactFlow<DataSourceNodeType>()
const { handleNodeDataUpdateWithSyncDraft } = useNodeDataUpdate()
const getNodeData = useCallback(() => {
const { getNodes } = store.getState()
const nodes = getNodes()
return nodes.find((node) => node.id === id)
}, [store, id])
}, [getNodes, id])
const handleNodeDataUpdate = useCallback(
(data: Partial<DataSourceNodeType>) => {
@@ -51,7 +57,7 @@ export const useConfig = (id: string, dataSourceList?: any[]) => {
)
const handleParametersChange = useCallback(
(datasource_parameters: ToolVarInputs) => {
(datasource_parameters: ResourceVarInputs) => {
const nodeData = getNodeData()
handleNodeDataUpdate({
...nodeData?.data,
@@ -62,59 +68,42 @@ export const useConfig = (id: string, dataSourceList?: any[]) => {
)
const outputSchema = useMemo(() => {
const nodeData = getNodeData()
if (!nodeData?.data || !dataSourceList) return []
if (!dataSourceList) return []
const currentDataSource = dataSourceList.find(
(ds: any) => ds.plugin_id === nodeData.data.plugin_id,
const provider = matchDataSource(dataSourceList, data)
const datasource = provider?.declaration.datasources?.find(
(item) => item.identity.name === data.datasource_name,
)
const currentDataSourceItem = currentDataSource?.tools?.find(
(tool: any) => tool.name === nodeData.data.datasource_name,
)
const output_schema = currentDataSourceItem?.output_schema
const properties = datasource?.output_schema?.properties
if (!properties || typeof properties !== 'object' || Array.isArray(properties)) return []
const res: any[] = []
if (!output_schema || !output_schema.properties) return res
Object.keys(output_schema.properties).forEach((outputKey) => {
const output = output_schema.properties[outputKey]
const type = output.type
if (type === 'object') {
res.push({
name: outputKey,
value: output,
})
} else {
res.push({
name: outputKey,
type:
output.type === 'array'
? `Array[${output.items?.type.slice(0, 1).toLocaleUpperCase()}${output.items?.type.slice(1)}]`
: `${output.type.slice(0, 1).toLocaleUpperCase()}${output.type.slice(1)}`,
description: output.description,
})
return Object.entries(properties).map(([name, value]: [string, unknown]) => {
const schema = value && typeof value === 'object' && !Array.isArray(value) ? value : {}
const type =
'type' in schema && typeof schema.type === 'string' && schema.type ? schema.type : 'unknown'
const items = 'items' in schema ? schema.items : undefined
const itemType =
items &&
typeof items === 'object' &&
'type' in items &&
typeof items.type === 'string' &&
items.type
? items.type
: 'unknown'
return {
name,
value,
type: type === 'array' ? `array[${itemType}]` : type,
description:
'description' in schema && typeof schema.description === 'string'
? schema.description
: '',
isObject: type === 'object',
}
})
return res
}, [getNodeData, dataSourceList])
}, [data, dataSourceList])
const hasObjectOutput = useMemo(() => {
const nodeData = getNodeData()
if (!nodeData?.data || !dataSourceList) return false
const currentDataSource = dataSourceList.find(
(ds: any) => ds.plugin_id === nodeData.data.plugin_id,
)
const currentDataSourceItem = currentDataSource?.tools?.find(
(tool: any) => tool.name === nodeData.data.datasource_name,
)
const output_schema = currentDataSourceItem?.output_schema
if (!output_schema || !output_schema.properties) return false
const properties = output_schema.properties
return Object.keys(properties).some((key) => properties[key].type === 'object')
}, [getNodeData, dataSourceList])
const hasObjectOutput = outputSchema.some((output) => output.isObject)
return {
handleFileExtensionsChange,

View File

@@ -4,12 +4,12 @@ import type { NodePanelProps } from '@/app/components/workflow/types'
import { memo, useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import TagInput from '@/app/components/base/tag-input'
import { toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema'
import { BoxGroupField } from '@/app/components/workflow/nodes/_base/components/layout'
import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/components/output-vars'
import StructureOutputItem from '@/app/components/workflow/nodes/_base/components/variable/object-child-tree-panel/show'
import { useStore } from '@/app/components/workflow/store'
import { wrapStructuredVarItem } from '@/app/components/workflow/utils/tool'
import { datasourceParametersToFormSchemas } from '@/app/components/workflow/utils/data-source'
import { matchDataSource } from '@/app/components/workflow/utils/plugin-install-check'
import { useNodesReadOnly } from '../../hooks/use-workflow'
import useMatchSchemaType, {
getMatchedSchemaType,
@@ -23,19 +23,18 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
const { t } = useTranslation()
const { nodesReadOnly } = useNodesReadOnly()
const dataSourceList = useStore((s) => s.dataSourceList)
const { provider_type, plugin_id, fileExtensions = [], datasource_parameters } = data
const { provider_type, fileExtensions = [], datasource_parameters } = data
const { handleFileExtensionsChange, handleParametersChange, outputSchema, hasObjectOutput } =
useConfig(id, dataSourceList)
useConfig(id, data, dataSourceList)
const isLocalFile = provider_type === DataSourceClassification.localFile
const currentDataSource = dataSourceList?.find((ds) => ds.plugin_id === plugin_id)
const currentDataSourceItem: any = currentDataSource?.tools?.find(
(tool: any) => tool.name === data.datasource_name,
const currentDataSource = matchDataSource(dataSourceList ?? [], data)
const currentDataSourceItem = currentDataSource?.declaration.datasources?.find(
(item) => item.identity.name === data.datasource_name,
)
const formSchemas = useMemo(
() => datasourceParametersToFormSchemas(currentDataSourceItem?.parameters),
[currentDataSourceItem],
)
const formSchemas = useMemo(() => {
return currentDataSourceItem
? toolParametersToFormSchemas(currentDataSourceItem.parameters)
: []
}, [currentDataSourceItem])
const pipelineId = useStore((s) => s.pipelineId)
const setShowInputFieldPanel = useStore((s) => s.setShowInputFieldPanel)
@@ -58,11 +57,10 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
<ToolForm
readOnly={nodesReadOnly}
nodeId={id}
schema={formSchemas as any}
schema={formSchemas}
staticSchema
value={datasource_parameters}
onChange={handleParametersChange}
currentProvider={currentDataSource}
currentTool={currentDataSourceItem}
showManageInputField={!!pipelineId}
onManageInputField={() => setShowInputFieldPanel?.(true)}
/>
@@ -119,14 +117,22 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
/>
))}
{outputSchema.map((outputItem) => {
const schemaType = getMatchedSchemaType(outputItem.value, schemaTypeDefinitions)
const schema =
outputItem.value &&
typeof outputItem.value === 'object' &&
!Array.isArray(outputItem.value)
? outputItem.value
: {}
const schemaType = getMatchedSchemaType(schema, schemaTypeDefinitions)
return (
<div key={outputItem.name}>
{outputItem.value?.type === 'object' ? (
{outputItem.isObject ? (
<StructureOutputItem
rootClassName="code-sm-semibold text-text-secondary"
payload={wrapStructuredVarItem(outputItem, schemaType)}
payload={{
schema: { properties: { [outputItem.name]: { ...schema, schemaType } } },
}}
/>
) : (
<VarItem

View File

@@ -9,8 +9,6 @@ export const DataSourceClassification = {
localFile: 'local_file',
} as const
export type ToolVarInputs = ResourceVarInputs
export type DataSourceNodeType = CommonNodeType & {
fileExtensions?: string[]
plugin_id: string
@@ -18,7 +16,7 @@ export type DataSourceNodeType = CommonNodeType & {
provider_name: string
datasource_name: string
datasource_label: string
datasource_parameters: ToolVarInputs
datasource_parameters: ResourceVarInputs
datasource_configurations: Record<string, any>
plugin_unique_identifier?: string
}

View File

@@ -1,15 +1,16 @@
'use client'
import type { FC } from 'react'
import type { CredentialFormSchema } from '@/app/components/header/account-setting/model-provider-page/declarations'
import type { Tool } from '@/app/components/tools/types'
import type { FormInputSchema } from '@/app/components/workflow/nodes/_base/components/form-input-item.helpers'
import type { ResourceVarInputs } from '@/app/components/workflow/nodes/_base/types'
import type { ToolWithProvider } from '@/app/components/workflow/types'
import ToolFormItem from './item'
type Props = Readonly<{
staticSchema?: boolean
readOnly: boolean
nodeId: string
schema: CredentialFormSchema[]
schema: FormInputSchema[]
value: ResourceVarInputs
onChange: (value: ResourceVarInputs) => void
onOpen?: (index: number) => void
@@ -23,6 +24,7 @@ type Props = Readonly<{
const ToolForm: FC<Props> = ({
readOnly,
staticSchema = false,
nodeId,
schema,
value,
@@ -40,6 +42,7 @@ const ToolForm: FC<Props> = ({
<ToolFormItem
key={index}
readOnly={readOnly}
staticSchema={staticSchema}
nodeId={nodeId}
schema={schema}
value={value}
@@ -50,7 +53,7 @@ const ToolForm: FC<Props> = ({
showManageInputField={showManageInputField}
onManageInputField={onManageInputField}
extraParams={extraParams}
providerType="tool"
providerType={staticSchema ? undefined : 'tool'}
/>
))}
</div>

View File

@@ -1,7 +1,7 @@
'use client'
import type { FC, ReactNode } from 'react'
import type { CredentialFormSchema } from '@/app/components/header/account-setting/model-provider-page/declarations'
import type { Tool } from '@/app/components/tools/types'
import type { FormInputSchema } from '@/app/components/workflow/nodes/_base/components/form-input-item.helpers'
import type { ResourceVarInputs } from '@/app/components/workflow/nodes/_base/types'
import type { ToolWithProvider } from '@/app/components/workflow/types'
import { Button } from '@langgenius/dify-ui/button'
@@ -49,9 +49,10 @@ const renderDescriptionWithLinks = (description: string): ReactNode => {
}
type Props = Readonly<{
staticSchema?: boolean
readOnly: boolean
nodeId: string
schema: CredentialFormSchema
schema: FormInputSchema
value: ResourceVarInputs
onChange: (value: ResourceVarInputs) => void
inPanel?: boolean
@@ -65,6 +66,7 @@ type Props = Readonly<{
const ToolFormItem: FC<Props> = ({
readOnly,
staticSchema = false,
nodeId,
schema,
value,
@@ -133,6 +135,7 @@ const ToolFormItem: FC<Props> = ({
<FormInputItem
labelId={labelId}
readOnly={readOnly}
staticSchema={staticSchema}
nodeId={nodeId}
schema={schema}
value={value}

View File

@@ -1,5 +1,5 @@
import type { TFunction } from 'i18next'
import type { NodeDefault, ToolWithProvider, Var } from '../../types'
import type { NodeDefault, ToolWithProvider, Var, WorkflowPluginCatalogs } from '../../types'
import type { ToolNodeType } from './types'
import { CollectionType } from '@/app/components/tools/types'
import { VarKindType } from '@/app/components/workflow/nodes/_base/types'
@@ -108,7 +108,7 @@ const nodeDefault: NodeDefault<ToolNodeType> = {
},
getOutputVars(
payload: ToolNodeType,
allPluginInfoList: Record<string, ToolWithProvider[]>,
allPluginInfoList: WorkflowPluginCatalogs,
_ragVars: any,
{ schemaTypeDefinitions } = { schemaTypeDefinitions: [] },
) {

View File

@@ -1,3 +1,4 @@
import type { RagPipelineDatasourceProviderResponse } from '@dify/contracts/api/console/rag/types.gen'
import type { Edge as ReactFlowEdge, Node as ReactFlowNode, Viewport, XYPosition } from 'reactflow'
import type { Plugin, PluginMeta } from '@/app/components/plugins/types'
import type { Collection, Tool } from '@/app/components/tools/types'
@@ -122,7 +123,12 @@ export type CommonNodeType<T = {}> = {
provider_id?: string
_dimmed?: boolean
} & T &
Partial<PluginDefaultValue>
Partial<
Pick<
PluginDefaultValue,
'provider_type' | 'provider_name' | 'plugin_id' | 'plugin_unique_identifier'
>
>
export type CommonEdgeType = {
_hovering?: boolean
@@ -353,6 +359,14 @@ export type NodeOutPutVar = {
isFlat?: boolean
}
export type WorkflowPluginCatalogs = {
buildInTools?: ToolWithProvider[]
customTools?: ToolWithProvider[]
workflowTools?: ToolWithProvider[]
mcpTools?: ToolWithProvider[]
dataSourceList?: RagPipelineDatasourceProviderResponse[]
}
export type NodeDefault<T = {}> = {
metaData: {
classification: BlockClassification
@@ -377,7 +391,7 @@ export type NodeDefault<T = {}> = {
) => { isValid: boolean; errorMessage?: string }
getOutputVars?: (
payload: T,
allPluginInfoList: Record<string, ToolWithProvider[]>,
allPluginInfoList: WorkflowPluginCatalogs,
ragVariables?: Var[],
utils?: {
schemaTypeDefinitions?: SchemaTypeDefinition[]

View File

@@ -1,131 +1,113 @@
import type { DatasourceParameter } from '@dify/contracts/api/console/rag/types.gen'
import type { DataSourceNodeType } from '../../nodes/data-source/types'
import type { ToolWithProvider } from '../../types'
import { FormTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { BlockEnum } from '../../types'
import { getDataSourceCheckParams } from '../data-source'
import { datasourceParametersToFormSchemas, getDataSourceCheckParams } from '../data-source'
vi.mock('@/app/components/tools/utils/to-form-schema', () => ({
toolParametersToFormSchemas: vi.fn((params: Array<Record<string, unknown>>) =>
params.map((p) => ({
variable: p.name,
label: p.label || { en_US: p.name },
type: p.type || 'string',
required: p.required ?? false,
form: p.form ?? 'llm',
hide: p.hide ?? false,
})),
),
}))
function createDataSourceData(overrides: Partial<DataSourceNodeType> = {}): DataSourceNodeType {
return {
title: 'DataSource',
desc: '',
type: BlockEnum.DataSource,
plugin_id: 'plugin-ds-1',
provider_type: 'local_file',
datasource_name: 'mysql_query',
datasource_parameters: {},
datasource_configurations: {},
...overrides,
} as DataSourceNodeType
const node: DataSourceNodeType = {
title: 'Datasource',
desc: '',
type: BlockEnum.DataSource,
plugin_id: 'langgenius/file',
provider_type: 'local_file',
provider_name: 'file',
datasource_name: 'local-file',
datasource_label: 'File',
datasource_parameters: {},
datasource_configurations: {},
}
const parameter = (
type: DatasourceParameter['type'],
defaultValue: DatasourceParameter['default'],
) =>
({
name: type,
type,
default: defaultValue,
label: { en_US: 'Parameter', zh_Hans: null },
description: { en_US: 'Actual datasource description', ja_JP: null },
required: true,
}) satisfies DatasourceParameter
function createDataSourceCollection(overrides: Partial<ToolWithProvider> = {}): ToolWithProvider {
return {
id: 'ds-collection',
plugin_id: 'plugin-ds-1',
name: 'MySQL',
tools: [
{
name: 'mysql_query',
parameters: [
{
name: 'query',
label: { en_US: 'SQL Query', zh_Hans: 'SQL 查询' },
type: 'string',
required: true,
},
{ name: 'limit', label: { en_US: 'Limit' }, type: 'number', required: false, hide: true },
],
},
],
allow_delete: true,
is_authorized: false,
...overrides,
} as unknown as ToolWithProvider
}
describe('getDataSourceCheckParams', () => {
it('should extract input schema from matching data source', () => {
const result = getDataSourceCheckParams(
createDataSourceData(),
[createDataSourceCollection()],
'en_US',
)
expect(result.dataSourceInputsSchema).toEqual([
{ label: 'SQL Query', variable: 'query', type: 'string', required: true, hide: false },
{ label: 'Limit', variable: 'limit', type: 'number', required: false, hide: true },
])
})
it('should not require authorization for a local file datasource', () => {
const result = getDataSourceCheckParams(
createDataSourceData(),
[createDataSourceCollection()],
'en_US',
)
expect(result.notAuthed).toBe(false)
})
it('should mark an unauthorized online datasource as not authed', () => {
const result = getDataSourceCheckParams(
createDataSourceData({ provider_type: 'online_document' }),
[createDataSourceCollection()],
'en_US',
)
expect(result.notAuthed).toBe(true)
})
it('should mark as authed when is_authorized is true', () => {
const result = getDataSourceCheckParams(
createDataSourceData({ provider_type: 'online_document' }),
[createDataSourceCollection({ is_authorized: true })],
'en_US',
)
expect(result.notAuthed).toBe(false)
})
it('should return empty schemas when data source is not found', () => {
const result = getDataSourceCheckParams(
createDataSourceData({ plugin_id: 'non-existent' }),
[createDataSourceCollection()],
'en_US',
)
expect(result.dataSourceInputsSchema).toEqual([])
})
it('should return empty schemas when datasource item is not found', () => {
const result = getDataSourceCheckParams(
createDataSourceData({ datasource_name: 'non_existent_ds' }),
[createDataSourceCollection()],
'en_US',
)
expect(result.dataSourceInputsSchema).toEqual([])
})
it('should include language in result', () => {
const result = getDataSourceCheckParams(
createDataSourceData(),
[createDataSourceCollection()],
'zh_Hans',
)
expect(result.language).toBe('zh_Hans')
it('projects datasource descriptions, nullable locale fallbacks, and exact default values at the form boundary', () => {
const schemas = datasourceParametersToFormSchemas([
{ ...parameter('number', 0), min: 0, max: 10, placeholder: { en_US: 'Count', zh_Hans: null } },
parameter('boolean', false),
parameter('string', ''),
parameter('secret-input', null),
{
...parameter('select', 'one'),
options: [{ value: 'one', label: { en_US: 'One', zh_Hans: null }, icon: null }],
},
parameter('system-files', []),
])
expect(schemas.map((schema) => schema.default)).toEqual([0, false, '', null, 'one', []])
expect(schemas[0]).toMatchObject({
type: FormTypeEnum.textNumber,
min: 0,
max: 10,
tooltip: { en_US: 'Actual datasource description', ja_JP: 'Actual datasource description' },
label: { zh_Hans: 'Parameter' },
placeholder: { zh_Hans: 'Count' },
})
expect(schemas[1]).toMatchObject({ type: FormTypeEnum.checkbox, _type: FormTypeEnum.boolean })
expect(schemas[4]?.options?.[0]).toMatchObject({ value: 'one', label: { zh_Hans: 'One' } })
expect(schemas[5]?.type).toBe(FormTypeEnum.files)
})
it('derives only required-input metadata and keeps local-file authorization independent', () => {
const provider = createDatasourceProvider({ is_authorized: false })
provider.declaration.datasources![0]!.parameters = [parameter('string', '')]
expect(getDataSourceCheckParams(node, [provider], 'zh_Hans')).toEqual({
dataSourceInputsSchema: [{ label: 'Parameter', variable: 'string', required: true }],
notAuthed: false,
language: 'zh_Hans',
})
expect(
getDataSourceCheckParams({ ...node, provider_type: 'online_document' }, [provider], 'en_US')
.notAuthed,
).toBe(true)
expect(
getDataSourceCheckParams({ ...node, provider_type: 'online_document' }, [], 'en_US').notAuthed,
).toBe(false)
})
it('finds a saved provider without version metadata and leaves missing datasource parameters empty', () => {
const provider = createDatasourceProvider()
expect(
getDataSourceCheckParams(
{ ...node, plugin_id: '', provider_name: provider.provider },
[provider],
'en_US',
).notAuthed,
).toBe(false)
expect(
getDataSourceCheckParams({ ...node, datasource_name: 'missing' }, [provider], 'en_US')
.dataSourceInputsSchema,
).toEqual([])
})
it('uses the saved plugin authorization and parameter schema when providers share a name', () => {
const official = createDatasourceProvider({ is_authorized: true })
official.declaration.datasources![0]!.parameters = [
{ ...parameter('string', ''), name: 'official_input' },
]
const fork = createDatasourceProvider({
plugin_id: 'acme/file',
plugin_unique_identifier: 'acme/file:1.0.0',
is_authorized: false,
})
fork.declaration.datasources![0]!.parameters = [
{ ...parameter('string', ''), name: 'fork_input' },
]
const result = getDataSourceCheckParams(
{ ...node, provider_type: 'online_document' },
[fork, official],
'en_US',
)
expect(result.notAuthed).toBe(false)
expect(result.dataSourceInputsSchema).toEqual([
{ label: 'Parameter', variable: 'official_input', required: true },
])
})

View File

@@ -1,5 +1,6 @@
import type { TriggerWithProvider } from '../../block-selector/types'
import type { CommonNodeType, ToolWithProvider } from '../../types'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { CollectionType } from '@/app/components/tools/types'
import { BlockEnum } from '../../types'
import {
@@ -72,7 +73,7 @@ describe('plugin install check', () => {
describe('matchDataSource', () => {
const dataSources = [
createTool({
createDatasourceProvider({
provider: 'knowledge-provider',
plugin_id: 'knowledge-plugin',
plugin_unique_identifier: 'knowledge-plugin@1.0.0',
@@ -96,6 +97,46 @@ describe('plugin install check', () => {
dataSources[0],
)
})
it('keeps plugin identity authoritative when another plugin has the same provider name', () => {
const fork = createDatasourceProvider({
provider: 'notion',
plugin_id: 'acme/notion',
plugin_unique_identifier: 'acme/notion:1.0.0',
is_authorized: false,
})
const official = createDatasourceProvider({
provider: 'notion',
plugin_id: 'langgenius/notion',
plugin_unique_identifier: 'langgenius/notion:2.0.0',
is_authorized: true,
})
const savedNode = {
plugin_id: official.plugin_id,
plugin_unique_identifier: official.plugin_unique_identifier,
provider_name: 'notion',
}
expect(matchDataSource([fork, official], savedNode)).toBe(official)
expect(
matchDataSource([fork, official], {
...savedNode,
plugin_unique_identifier: 'langgenius/notion:1.0.0',
}),
).toBe(official)
expect(matchDataSource([fork], savedNode)).toBeUndefined()
expect(
matchDataSource([fork], {
plugin_unique_identifier: official.plugin_unique_identifier,
provider_name: 'notion',
}),
).toBeUndefined()
expect(
isNodePluginMissing(
{ ...savedNode, type: BlockEnum.DataSource, title: 'Notion', desc: '' },
{ dataSourceList: [fork] },
),
).toBe(true)
})
})
describe('isNodePluginMissing', () => {
@@ -170,7 +211,7 @@ describe('plugin install check', () => {
plugin_unique_identifier: 'missing-data-source@1.0.0',
} as CommonNodeType
expect(isNodePluginMissing(node, { dataSourceList: [createTool()] })).toBe(true)
expect(isNodePluginMissing(node, { dataSourceList: [createDatasourceProvider()] })).toBe(true)
})
it('should keep data source nodes installable when the list has not loaded yet', () => {

View File

@@ -1,40 +1,79 @@
import type {
DatasourceParameter,
I18nObject,
RagPipelineDatasourceProviderResponse,
} from '@dify/contracts/api/console/rag/types.gen'
import type { FormInputSchema } from '../nodes/_base/components/form-input-item.helpers'
import type { DataSourceNodeType } from '../nodes/data-source/types'
import type { InputVar, ToolWithProvider } from '../types'
import { toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema'
import type { TypeWithI18N } from '@/app/components/header/account-setting/model-provider-page/declarations'
import { FormTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
import { DataSourceClassification } from '../nodes/data-source/types'
import { matchDataSource } from './plugin-install-check'
const toFormLabel = (label: I18nObject): TypeWithI18N => ({
en_US: label.en_US,
zh_Hans: label.zh_Hans ?? label.en_US,
ja_JP: label.ja_JP ?? label.en_US,
pt_BR: label.pt_BR ?? label.en_US,
})
export const datasourceParametersToFormSchemas = (
parameters: DatasourceParameter[] = [],
): FormInputSchema[] =>
parameters.map((parameter) => {
const types = {
string: FormTypeEnum.textInput,
number: FormTypeEnum.textNumber,
boolean: FormTypeEnum.checkbox,
select: FormTypeEnum.select,
'secret-input': FormTypeEnum.secretInput,
file: FormTypeEnum.file,
files: FormTypeEnum.files,
'system-files': FormTypeEnum.files,
}
return {
name: parameter.name,
variable: parameter.name,
label: toFormLabel(parameter.label),
type: types[parameter.type],
_type: parameter.type === 'boolean' ? FormTypeEnum.boolean : types[parameter.type],
required: parameter.required ?? false,
default: parameter.default,
tooltip: toFormLabel(parameter.description),
placeholder: parameter.placeholder ? toFormLabel(parameter.placeholder) : undefined,
min: parameter.min ?? undefined,
max: parameter.max ?? undefined,
scope: parameter.scope ?? undefined,
show_on: [],
options: parameter.options?.map((option) => ({
label: toFormLabel(option.label),
value: option.value,
icon: option.icon ?? undefined,
show_on: [],
})),
}
})
export const getDataSourceCheckParams = (
toolData: DataSourceNodeType,
dataSourceList: ToolWithProvider[],
data: DataSourceNodeType,
dataSourceList: RagPipelineDatasourceProviderResponse[],
language: string,
) => {
const { plugin_id, provider_type, datasource_name } = toolData
const currentDataSource = dataSourceList.find((item) => item.plugin_id === plugin_id)
const currentDataSourceItem = currentDataSource?.tools.find(
(tool) => tool.name === datasource_name,
const currentDataSource = matchDataSource(dataSourceList, data)
const datasource = currentDataSource?.declaration.datasources?.find(
(item) => item.identity.name === data.datasource_name,
)
const formSchemas = currentDataSourceItem
? toolParametersToFormSchemas(currentDataSourceItem.parameters)
: []
const formSchemas = datasourceParametersToFormSchemas(datasource?.parameters)
return {
dataSourceInputsSchema: (() => {
const formInputs: InputVar[] = []
formSchemas.forEach((item: any) => {
formInputs.push({
label: item.label[language] || item.label.en_US,
variable: item.variable,
type: item.type,
required: item.required,
hide: item.hide,
})
})
return formInputs
})(),
dataSourceInputsSchema: formSchemas.map((item) => ({
label: item.label[language] || item.label.en_US,
variable: item.variable,
required: item.required,
})),
notAuthed:
provider_type !== DataSourceClassification.localFile &&
!!currentDataSource?.allow_delete &&
!currentDataSource?.is_authorized,
data.provider_type !== DataSourceClassification.localFile &&
!!currentDataSource &&
!currentDataSource.is_authorized,
language,
}
}

View File

@@ -1,3 +1,4 @@
import type { RagPipelineDatasourceProviderResponse } from '@dify/contracts/api/console/rag/types.gen'
import type { TriggerWithProvider } from '../block-selector/types'
import type { DataSourceNodeType } from '../nodes/data-source/types'
import type { ToolNodeType } from '../nodes/tool/types'
@@ -32,16 +33,18 @@ export function matchTriggerProvider(
}
export function matchDataSource(
list: ToolWithProvider[],
list: RagPipelineDatasourceProviderResponse[],
data: { plugin_unique_identifier?: string; plugin_id?: string; provider_name?: string },
): ToolWithProvider | undefined {
return list.find(
(item) =>
(data.plugin_unique_identifier &&
item.plugin_unique_identifier === data.plugin_unique_identifier) ||
(data.plugin_id && item.plugin_id === data.plugin_id) ||
(data.provider_name && item.provider === data.provider_name),
)
): RagPipelineDatasourceProviderResponse | undefined {
if (data.plugin_unique_identifier) {
const installedVersion = list.find(
(item) => item.plugin_unique_identifier === data.plugin_unique_identifier,
)
if (installedVersion) return installedVersion
}
if (data.plugin_id) return list.find((item) => item.plugin_id === data.plugin_id)
if (data.plugin_unique_identifier) return undefined
return list.find((item) => item.provider === data.provider_name)
}
type PluginInstallCheckContext = {
@@ -50,7 +53,7 @@ type PluginInstallCheckContext = {
workflowTools?: ToolWithProvider[]
mcpTools?: ToolWithProvider[]
triggerPlugins?: TriggerWithProvider[]
dataSourceList?: ToolWithProvider[]
dataSourceList?: RagPipelineDatasourceProviderResponse[]
}
export function isNodePluginMissing(

View File

@@ -200,7 +200,6 @@
"openInExplore": "فتح Web App في الشريط الجانبي",
"publishApp.notSet": "لم يتم تعيينه",
"removeOriginal": "حذف التطبيق الأصلي",
"structOutput.LLMResponse": "استجابة LLM",
"structOutput.configure": "تكوين",
"structOutput.modelNotSupported": "النموذج غير مدعوم",
"structOutput.modelNotSupportedTip": "النموذج الحالي لا يدعم هذه الميزة ويتم تخفيضه تلقائيًا إلى حقن المطالبة.",

View File

@@ -200,7 +200,6 @@
"openInExplore": "Yan paneldəki veb tətbiqini aç",
"publishApp.notSet": "Təyin edilməyib",
"removeOriginal": "Orijinal tətbiqi sil",
"structOutput.LLMResponse": "LLM cavabı",
"structOutput.configure": "Konfiqurasiya et",
"structOutput.modelNotSupported": "Model dəstəklənmir",
"structOutput.modelNotSupportedTip": "Cari model bu funksiyanı dəstəkləmir və avtomatik olaraq təlimatın əlavə edilməsi üsuluna keçirilir.",

View File

@@ -200,7 +200,6 @@
"openInExplore": "Web App in der Seitenleiste öffnen",
"publishApp.notSet": "Nicht festgelegt",
"removeOriginal": "Ursprüngliche App löschen",
"structOutput.LLMResponse": "LLM-Antwort",
"structOutput.configure": "Konfigurieren",
"structOutput.modelNotSupported": "Modell nicht unterstützt",
"structOutput.modelNotSupportedTip": "Das aktuelle Modell unterstützt diese Funktion nicht und wird automatisch auf Eingabeinjektion heruntergestuft.",

View File

@@ -200,7 +200,6 @@
"openInExplore": "Open Sidebar Web App",
"publishApp.notSet": "Not set",
"removeOriginal": "Delete the original app",
"structOutput.LLMResponse": "LLM Response",
"structOutput.configure": "Configure",
"structOutput.modelNotSupported": "Model not supported",
"structOutput.modelNotSupportedTip": "The current model does not support this feature and is automatically downgraded to prompt injection.",

View File

@@ -200,7 +200,6 @@
"openInExplore": "Abrir Web App de la barra lateral",
"publishApp.notSet": "No establecido",
"removeOriginal": "Eliminar la app original",
"structOutput.LLMResponse": "Respuesta del LLM",
"structOutput.configure": "Configurar",
"structOutput.modelNotSupported": "Modelo no soportado",
"structOutput.modelNotSupportedTip": "El modelo actual no admite esta función y se degrada automáticamente a inyección de comandos.",

View File

@@ -200,7 +200,6 @@
"openInExplore": "Web App نوار کناری را باز کنید",
"publishApp.notSet": "تنظیم نشده است",
"removeOriginal": "حذف برنامه اصلی",
"structOutput.LLMResponse": "پاسخ مدل زبان بزرگ",
"structOutput.configure": "تنظیمات",
"structOutput.modelNotSupported": "مدل پشتیبانی نمی شود",
"structOutput.modelNotSupportedTip": "مدل فعلی این ویژگی را پشتیبانی نمی‌کند و به‌طور خودکار به تزریق درخواست تنزل پیدا می‌کند.",

View File

@@ -200,7 +200,6 @@
"openInExplore": "Ouvrir la Web App de la barre latérale",
"publishApp.notSet": "Non défini",
"removeOriginal": "Supprimer l'application d'origine",
"structOutput.LLMResponse": "Réponse LLM",
"structOutput.configure": "Configurer",
"structOutput.modelNotSupported": "Modèle non pris en charge",
"structOutput.modelNotSupportedTip": "Le modèle actuel ne prend pas en charge cette fonctionnalité et est automatiquement rétrogradé à l'injection de prompt.",

View File

@@ -200,7 +200,6 @@
"openInExplore": "साइडबार Web App खोलें",
"publishApp.notSet": "अनुबंधित नहीं किया गया",
"removeOriginal": "मूल ऐप हटाएँ",
"structOutput.LLMResponse": "LLM प्रतिक्रिया",
"structOutput.configure": "कॉन्फ़िगर करें",
"structOutput.modelNotSupported": "मॉडल का समर्थन नहीं किया गया",
"structOutput.modelNotSupportedTip": "वर्तमान मॉडल इस सुविधा का समर्थन नहीं करता है और स्वचालित रूप से प्रॉम्प्ट इंजेक्शन में डाउनग्रेड किया जाता है।",

Some files were not shown because too many files have changed in this diff Show More