feat: support host style in archive s3 client (#43066)

api/.env.example

@@ -156,6 +156,7 @@ ARCHIVE_STORAGE_EXPORT_BUCKET=
 ARCHIVE_STORAGE_ACCESS_KEY=
 ARCHIVE_STORAGE_SECRET_KEY=
 ARCHIVE_STORAGE_REGION=auto
+ARCHIVE_STORAGE_ADDRESS_STYLE=path
 
 # Azure Blob Storage configuration
 AZURE_BLOB_ACCOUNT_NAME=your-account-name

api/configs/extra/archive_config.py

@@ -1,3 +1,5 @@
+from typing import Literal
+
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -41,3 +43,8 @@ class ArchiveStorageConfig(BaseSettings):
         description="Region for storage (use 'auto' if the provider supports it)",
         default="auto",
     )
+
+    ARCHIVE_STORAGE_ADDRESS_STYLE: Literal["path", "virtual", "auto"] = Field(
+        description="S3 addressing style for archive storage: 'path' (default), 'virtual' (virtual-hosted), or 'auto'",
+        default="path",
+    )

api/libs/archive_storage.py

@@ -77,7 +77,7 @@ class ArchiveStorage:
             aws_secret_access_key=dify_config.ARCHIVE_STORAGE_SECRET_KEY,
             region_name=dify_config.ARCHIVE_STORAGE_REGION,
             config=Config(
-                s3={"addressing_style": "path"},
+                s3={"addressing_style": dify_config.ARCHIVE_STORAGE_ADDRESS_STYLE},
                 max_pool_connections=64,
             ),
         )

api/tests/unit_tests/libs/test_archive_storage.py

@@ -4,8 +4,10 @@ from datetime import datetime
 from unittest.mock import ANY, MagicMock
 
 import pytest
+from botocore.awsrequest import AWSResponse
 from botocore.exceptions import ClientError, EndpointConnectionError
 
+from configs.extra.archive_config import ArchiveStorageConfig
 from libs import archive_storage as storage_module
 from libs.archive_storage import (
     ArchiveStorage,
@@ -117,6 +119,118 @@ def test_init_sets_client(monkeypatch: pytest.MonkeyPatch):
     assert storage.bucket == BUCKET_NAME
 
 
+def test_init_default_address_style_is_path(monkeypatch: pytest.MonkeyPatch):
+    # _configure_storage does not set ARCHIVE_STORAGE_ADDRESS_STYLE, so the
+    # pydantic field default is exercised.
+    _configure_storage(monkeypatch)
+    _, boto_client = _mock_client(monkeypatch)
+
+    ArchiveStorage(bucket=BUCKET_NAME)
+
+    assert ArchiveStorageConfig.model_fields["ARCHIVE_STORAGE_ADDRESS_STYLE"].default == "path"
+    config = boto_client.call_args.kwargs["config"]
+    assert config.s3 == {"addressing_style": "path"}
+
+
+@pytest.mark.parametrize("address_style", ["virtual", "auto"])
+def test_init_address_style_override(monkeypatch: pytest.MonkeyPatch, address_style: str):
+    _configure_storage(monkeypatch, ARCHIVE_STORAGE_ADDRESS_STYLE=address_style)
+    _, boto_client = _mock_client(monkeypatch)
+
+    ArchiveStorage(bucket=BUCKET_NAME)
+
+    config = boto_client.call_args.kwargs["config"]
+    assert config.s3 == {"addressing_style": address_style}
+
+
+class _EmptyRawBody:
+    def stream(self, **_kwargs):
+        yield b""
+
+
+def _record_real_client_requests(monkeypatch: pytest.MonkeyPatch) -> list[str]:
+    urls: list[str] = []
+    real_client = storage_module.boto3.client
+
+    def make_client(*args, **kwargs):
+        client = real_client(*args, **kwargs)
+
+        def handler(request, **_kwargs):
+            urls.append(request.url)
+            return AWSResponse(request.url, 200, {}, _EmptyRawBody())
+
+        client.meta.events.register("before-send.s3", handler)
+        return client
+
+    monkeypatch.setattr(storage_module.boto3, "client", make_client)
+    return urls
+
+
+@pytest.mark.parametrize(
+    ("address_style", "expected_bucket_url", "expected_object_url"),
+    [
+        (
+            None,
+            "https://account123.r2.example.com/archive-bucket",
+            "https://account123.r2.example.com/archive-bucket/some/key.json",
+        ),
+        (
+            "path",
+            "https://account123.r2.example.com/archive-bucket",
+            "https://account123.r2.example.com/archive-bucket/some/key.json",
+        ),
+        (
+            "virtual",
+            "https://archive-bucket.account123.r2.example.com/",
+            "https://archive-bucket.account123.r2.example.com/some/key.json",
+        ),
+    ],
+)
+def test_real_client_request_urls_follow_address_style(
+    monkeypatch: pytest.MonkeyPatch,
+    address_style: str | None,
+    expected_bucket_url: str,
+    expected_object_url: str,
+):
+    overrides: dict[str, object] = {"ARCHIVE_STORAGE_ENDPOINT": "https://account123.r2.example.com"}
+    if address_style is not None:
+        overrides["ARCHIVE_STORAGE_ADDRESS_STYLE"] = address_style
+    _configure_storage(monkeypatch, **overrides)
+    urls = _record_real_client_requests(monkeypatch)
+
+    storage = ArchiveStorage(bucket=BUCKET_NAME)
+    storage.object_exists("some/key.json")
+
+    assert len(urls) == 2
+    assert urls[0].startswith(expected_bucket_url)
+    assert urls[1].startswith(expected_object_url)
+
+
+@pytest.mark.parametrize(
+    ("address_style", "expected_prefix"),
+    [
+        (None, "https://account123.r2.example.com/archive-bucket/some/key.json?"),
+        ("path", "https://account123.r2.example.com/archive-bucket/some/key.json?"),
+        ("virtual", "https://archive-bucket.account123.r2.example.com/some/key.json?"),
+    ],
+)
+def test_real_client_presigned_url_follows_address_style(
+    monkeypatch: pytest.MonkeyPatch,
+    address_style: str | None,
+    expected_prefix: str,
+):
+    overrides: dict[str, object] = {"ARCHIVE_STORAGE_ENDPOINT": "https://account123.r2.example.com"}
+    if address_style is not None:
+        overrides["ARCHIVE_STORAGE_ADDRESS_STYLE"] = address_style
+    _configure_storage(monkeypatch, **overrides)
+    _record_real_client_requests(monkeypatch)
+
+    storage = ArchiveStorage(bucket=BUCKET_NAME)
+    url = storage.generate_presigned_url("some/key.json", expires_in=123)
+
+    assert url.startswith(expected_prefix)
+
+
 def test_put_object_returns_checksum(monkeypatch: pytest.MonkeyPatch):
     _configure_storage(monkeypatch)
     client, _ = _mock_client(monkeypatch)

docker/envs/core-services/shared.env.example

@@ -349,6 +349,7 @@ ARCHIVE_STORAGE_ENDPOINT=
 ARCHIVE_STORAGE_ARCHIVE_BUCKET=
 ARCHIVE_STORAGE_EXPORT_BUCKET=
 ARCHIVE_STORAGE_REGION=auto
+ARCHIVE_STORAGE_ADDRESS_STYLE=path
 AZURE_BLOB_ACCOUNT_NAME=difyai
 AZURE_BLOB_CONTAINER_NAME=difyai-container
 AZURE_KEYVAULT_VAULT_URL=