feat(dbspec): Support apache datafusion dbspec (#42374)
This commit is contained in:
Bhargava Vadlamani
committed by
GitHub
GitHub
parent
3bdf134aaa
commit
7eceebce1c
@@ -1,16 +1,16 @@
|
||||
{
|
||||
"generated": "2026-04-17T17:08:35.132Z",
|
||||
"statistics": {
|
||||
"totalDatabases": 80,
|
||||
"withDocumentation": 80,
|
||||
"withConnectionString": 80,
|
||||
"withDrivers": 28,
|
||||
"totalDatabases": 81,
|
||||
"withDocumentation": 81,
|
||||
"withConnectionString": 81,
|
||||
"withDrivers": 29,
|
||||
"withAuthMethods": 5,
|
||||
"supportsJoins": 74,
|
||||
"supportsSubqueries": 77,
|
||||
"supportsJoins": 75,
|
||||
"supportsSubqueries": 78,
|
||||
"supportsDynamicSchema": 29,
|
||||
"supportsCatalog": 20,
|
||||
"averageScore": 54,
|
||||
"averageScore": 53,
|
||||
"maxScore": 201,
|
||||
"byCategory": {
|
||||
"Cloud - AWS": [
|
||||
@@ -20,6 +20,7 @@
|
||||
],
|
||||
"Query Engines": [
|
||||
"Amazon Athena",
|
||||
"Apache DataFusion",
|
||||
"Apache Drill",
|
||||
"Apache Hive",
|
||||
"Apache Impala",
|
||||
@@ -94,6 +95,7 @@
|
||||
"Vertica"
|
||||
],
|
||||
"Apache Projects": [
|
||||
"Apache DataFusion",
|
||||
"Apache Doris",
|
||||
"Apache Drill",
|
||||
"Apache Druid",
|
||||
@@ -108,6 +110,7 @@
|
||||
"Databricks Interactive Cluster"
|
||||
],
|
||||
"Open Source": [
|
||||
"Apache DataFusion",
|
||||
"Apache Doris",
|
||||
"Apache Drill",
|
||||
"Apache Druid",
|
||||
@@ -9698,6 +9701,97 @@
|
||||
"supports_dynamic_schema": true,
|
||||
"supports_catalog": true,
|
||||
"supports_dynamic_catalog": true
|
||||
},
|
||||
"Apache DataFusion": {
|
||||
"time_grains": {
|
||||
"SECOND": true,
|
||||
"FIVE_SECONDS": true,
|
||||
"THIRTY_SECONDS": true,
|
||||
"MINUTE": true,
|
||||
"FIVE_MINUTES": true,
|
||||
"TEN_MINUTES": true,
|
||||
"FIFTEEN_MINUTES": true,
|
||||
"THIRTY_MINUTES": true,
|
||||
"HALF_HOUR": false,
|
||||
"HOUR": true,
|
||||
"SIX_HOURS": false,
|
||||
"DAY": true,
|
||||
"WEEK": true,
|
||||
"WEEK_STARTING_SUNDAY": false,
|
||||
"WEEK_STARTING_MONDAY": false,
|
||||
"WEEK_ENDING_SATURDAY": false,
|
||||
"WEEK_ENDING_SUNDAY": false,
|
||||
"MONTH": true,
|
||||
"QUARTER": true,
|
||||
"QUARTER_YEAR": false,
|
||||
"YEAR": true
|
||||
},
|
||||
"module": "superset.db_engine_specs.datafusion",
|
||||
"limit_method": 1,
|
||||
"limit_clause": true,
|
||||
"joins": true,
|
||||
"subqueries": true,
|
||||
"alias_in_select": true,
|
||||
"alias_in_orderby": true,
|
||||
"time_groupby_inline": false,
|
||||
"alias_to_source_column": false,
|
||||
"order_by_not_in_select": true,
|
||||
"expressions_in_orderby": false,
|
||||
"cte_in_subquery": true,
|
||||
"max_column_name": null,
|
||||
"sql_comments": true,
|
||||
"escaped_colons": true,
|
||||
"masked_encrypted_extra": false,
|
||||
"column_type_mapping": false,
|
||||
"function_names": false,
|
||||
"user_impersonation": false,
|
||||
"file_upload": false,
|
||||
"get_extra_table_metadata": false,
|
||||
"dbapi_exception_mapping": false,
|
||||
"custom_errors": false,
|
||||
"dynamic_schema": false,
|
||||
"catalog": false,
|
||||
"dynamic_catalog": false,
|
||||
"ssh_tunneling": true,
|
||||
"query_cancelation": false,
|
||||
"get_metrics": false,
|
||||
"where_latest_partition": false,
|
||||
"expand_data": false,
|
||||
"query_cost_estimation": false,
|
||||
"sql_validation": false,
|
||||
"score": 24,
|
||||
"max_score": 201,
|
||||
"documentation": {
|
||||
"description": "DataFusion is a highly performant query engine",
|
||||
"logo": "datafusion.png",
|
||||
"homepage_url": "https://datafusion.apache.org/",
|
||||
"categories": [
|
||||
"Query Engines",
|
||||
"Open Source",
|
||||
"Apache Projects"
|
||||
],
|
||||
"pypi_packages": [
|
||||
"flightsql-dbapi"
|
||||
],
|
||||
"connection_string": "datafusion://host:port",
|
||||
"drivers": [
|
||||
{
|
||||
"name": "Arrow Flight SQL (Recommended)",
|
||||
"pypi_package": "flightsql-dbapi",
|
||||
"connection_string": "datafusion://host:port",
|
||||
"is_recommended": true
|
||||
}
|
||||
],
|
||||
"category": "Apache Projects"
|
||||
},
|
||||
"engine": "datafusion",
|
||||
"engine_name": "Apache DataFusion",
|
||||
"engine_aliases": [],
|
||||
"default_driver": "flightsql",
|
||||
"supports_file_upload": false,
|
||||
"supports_dynamic_schema": false,
|
||||
"supports_catalog": false,
|
||||
"supports_dynamic_catalog": false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
BIN
docs/static/img/databases/datafusion.png
vendored
Normal file
BIN
docs/static/img/databases/datafusion.png
vendored
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
@@ -142,6 +142,7 @@ databricks = [
|
||||
"databricks-sql-connector>=4.2.6, <4.4.0",
|
||||
"databricks-sqlalchemy==1.0.5",
|
||||
]
|
||||
datafusion = ["flightsql-dbapi>=0.2.0, <0.3"]
|
||||
db2 = ["ibm-db-sa>0.3.8, <=0.4.4"]
|
||||
denodo = ["denodo-sqlalchemy>=2.0.5,<2.1.0"]
|
||||
dremio = ["sqlalchemy-dremio>=1.2.1, <4"]
|
||||
|
||||
92
superset/db_engine_specs/datafusion.py
Normal file
92
superset/db_engine_specs/datafusion.py
Normal file
@@ -0,0 +1,92 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import types
|
||||
|
||||
from superset.constants import TimeGrain
|
||||
from superset.db_engine_specs import BaseEngineSpec
|
||||
from superset.db_engine_specs.base import DatabaseCategory
|
||||
|
||||
|
||||
class DataFusionEngineSpec(BaseEngineSpec):
|
||||
engine_name = "Apache DataFusion"
|
||||
engine = "datafusion"
|
||||
drivers = {
|
||||
"flightsql": "Arrow Flight SQL protocol for DataFusion",
|
||||
}
|
||||
default_driver = "flightsql"
|
||||
sqlalchemy_uri_placeholder = "datafusion://host:port"
|
||||
supports_file_upload = False
|
||||
|
||||
metadata = {
|
||||
"description": "DataFusion is a highly performant query engine",
|
||||
"logo": "datafusion.png",
|
||||
"homepage_url": "https://datafusion.apache.org/",
|
||||
"categories": [
|
||||
DatabaseCategory.QUERY_ENGINES,
|
||||
DatabaseCategory.OPEN_SOURCE,
|
||||
DatabaseCategory.APACHE_PROJECTS,
|
||||
],
|
||||
"pypi_packages": ["flightsql-dbapi"],
|
||||
"connection_string": "datafusion://host:port",
|
||||
"drivers": [
|
||||
{
|
||||
"name": "Arrow Flight SQL (Recommended)",
|
||||
"pypi_package": "flightsql-dbapi",
|
||||
"connection_string": "datafusion://host:port",
|
||||
"is_recommended": True,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
_time_grain_expressions = {
|
||||
None: "{col}",
|
||||
TimeGrain.SECOND: "DATE_TRUNC('second', {col})",
|
||||
TimeGrain.FIVE_SECONDS: "DATE_TRUNC('minute', {col}) + INTERVAL '5 seconds' * FLOOR(EXTRACT(SECOND FROM {col}) / 5)", # noqa: E501
|
||||
TimeGrain.THIRTY_SECONDS: "DATE_TRUNC('minute', {col}) + INTERVAL '30 seconds' * FLOOR(EXTRACT(SECOND FROM {col}) / 30)", # noqa: E501
|
||||
TimeGrain.MINUTE: "DATE_TRUNC('minute', {col})",
|
||||
TimeGrain.FIVE_MINUTES: "DATE_TRUNC('hour', {col}) + INTERVAL '5 minutes' * FLOOR(EXTRACT(MINUTE FROM {col}) / 5)", # noqa: E501
|
||||
TimeGrain.TEN_MINUTES: "DATE_TRUNC('hour', {col}) + INTERVAL '10 minutes' * FLOOR(EXTRACT(MINUTE FROM {col}) / 10)", # noqa: E501
|
||||
TimeGrain.FIFTEEN_MINUTES: "DATE_TRUNC('hour', {col}) + INTERVAL '15 minutes' * FLOOR(EXTRACT(MINUTE FROM {col}) / 15)", # noqa: E501
|
||||
TimeGrain.THIRTY_MINUTES: "DATE_TRUNC('hour', {col}) + INTERVAL '30 minutes' * FLOOR(EXTRACT(MINUTE FROM {col}) / 30)", # noqa: E501
|
||||
TimeGrain.HOUR: "DATE_TRUNC('hour', {col})",
|
||||
TimeGrain.DAY: "DATE_TRUNC('day', {col})",
|
||||
TimeGrain.WEEK: "DATE_TRUNC('week', {col})",
|
||||
TimeGrain.MONTH: "DATE_TRUNC('month', {col})",
|
||||
TimeGrain.QUARTER: "DATE_TRUNC('quarter', {col})",
|
||||
TimeGrain.YEAR: "DATE_TRUNC('year', {col})",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def epoch_to_dttm(cls) -> str:
|
||||
return "from_unixtime({col})"
|
||||
|
||||
@classmethod
|
||||
def convert_dttm(
|
||||
cls, target_type: str, dttm: datetime, db_extra: dict[str, Any] | None = None
|
||||
) -> str | None:
|
||||
sqla_type = cls.get_sqla_column_type(target_type)
|
||||
|
||||
if isinstance(sqla_type, types.Date):
|
||||
return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')"
|
||||
if isinstance(sqla_type, types.DateTime):
|
||||
dttm_formatted = dttm.isoformat(sep=" ", timespec="milliseconds")
|
||||
return f"""TO_TIMESTAMP('{dttm_formatted}', 'YYYY-MM-DD HH24:MI:SS.FFF')"""
|
||||
return None
|
||||
@@ -19,6 +19,7 @@
|
||||
# To set the images of your preferred database, you may create a mapping here with engine and locations of the relevant images. The image can be hosted locally inside your static/file directory or online (e.g. S3).
|
||||
|
||||
# DB_IMAGES:
|
||||
# datafusion: "path/to/image/datafusion.jpg"
|
||||
# postgresql: "path/to/image/postgres.jpg"
|
||||
# bigquery: "path/to/s3bucket/bigquery.jpg"
|
||||
# snowflake: "path/to/image/snowflake.jpg"
|
||||
|
||||
101
tests/unit_tests/db_engine_specs/test_datafusion.py
Normal file
101
tests/unit_tests/db_engine_specs/test_datafusion.py
Normal file
@@ -0,0 +1,101 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from superset.db_engine_specs.datafusion import DataFusionEngineSpec
|
||||
from tests.unit_tests.db_engine_specs.utils import assert_convert_dttm
|
||||
from tests.unit_tests.fixtures.common import dttm # noqa: F401
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target_type,expected_result",
|
||||
[
|
||||
("Date", "TO_DATE('2019-01-02', 'YYYY-MM-DD')"),
|
||||
(
|
||||
"DateTime",
|
||||
"TO_TIMESTAMP('2019-01-02 03:04:05.678', 'YYYY-MM-DD HH24:MI:SS.FFF')",
|
||||
),
|
||||
(
|
||||
"TimeStamp",
|
||||
"TO_TIMESTAMP('2019-01-02 03:04:05.678', 'YYYY-MM-DD HH24:MI:SS.FFF')",
|
||||
),
|
||||
("UnknownType", None),
|
||||
],
|
||||
)
|
||||
def test_convert_dttm(
|
||||
target_type: str,
|
||||
expected_result: str | None,
|
||||
dttm: datetime, # noqa: F811
|
||||
) -> None:
|
||||
assert_convert_dttm(DataFusionEngineSpec, target_type, expected_result, dttm)
|
||||
|
||||
|
||||
def test_epoch_to_dttm() -> None:
|
||||
assert DataFusionEngineSpec.epoch_to_dttm().format(col="ts") == "from_unixtime(ts)"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"time_grain,expected",
|
||||
[
|
||||
(None, "ts"),
|
||||
("PT1S", "DATE_TRUNC('second', ts)"),
|
||||
(
|
||||
"PT5S",
|
||||
"DATE_TRUNC('minute', ts) + INTERVAL '5 seconds' * "
|
||||
"FLOOR(EXTRACT(SECOND FROM ts) / 5)",
|
||||
),
|
||||
(
|
||||
"PT30S",
|
||||
"DATE_TRUNC('minute', ts) + INTERVAL '30 seconds' * "
|
||||
"FLOOR(EXTRACT(SECOND FROM ts) / 30)",
|
||||
),
|
||||
("PT1M", "DATE_TRUNC('minute', ts)"),
|
||||
(
|
||||
"PT5M",
|
||||
"DATE_TRUNC('hour', ts) + INTERVAL '5 minutes' * "
|
||||
"FLOOR(EXTRACT(MINUTE FROM ts) / 5)",
|
||||
),
|
||||
(
|
||||
"PT10M",
|
||||
"DATE_TRUNC('hour', ts) + INTERVAL '10 minutes' * "
|
||||
"FLOOR(EXTRACT(MINUTE FROM ts) / 10)",
|
||||
),
|
||||
(
|
||||
"PT15M",
|
||||
"DATE_TRUNC('hour', ts) + INTERVAL '15 minutes' * "
|
||||
"FLOOR(EXTRACT(MINUTE FROM ts) / 15)",
|
||||
),
|
||||
(
|
||||
"PT30M",
|
||||
"DATE_TRUNC('hour', ts) + INTERVAL '30 minutes' * "
|
||||
"FLOOR(EXTRACT(MINUTE FROM ts) / 30)",
|
||||
),
|
||||
("PT1H", "DATE_TRUNC('hour', ts)"),
|
||||
("P1D", "DATE_TRUNC('day', ts)"),
|
||||
("P1W", "DATE_TRUNC('week', ts)"),
|
||||
("P1M", "DATE_TRUNC('month', ts)"),
|
||||
("P3M", "DATE_TRUNC('quarter', ts)"),
|
||||
("P1Y", "DATE_TRUNC('year', ts)"),
|
||||
],
|
||||
)
|
||||
def test_time_grain_expressions(time_grain: str | None, expected: str) -> None:
|
||||
assert (
|
||||
DataFusionEngineSpec._time_grain_expressions[time_grain].format(col="ts")
|
||||
== expected
|
||||
)
|
||||
Reference in New Issue
Block a user