Compare commits

...

13 Commits

Author SHA1 Message Date
Callum Herries
d4600d439d stick an upterm session in there 2023-04-07 10:10:06 +08:00
Tim Macdonald
dee7729323 More test fixing 2023-04-06 15:56:17 +01:00
Tim Macdonald
b644d4ef16 test fix 2023-04-06 15:04:24 +01:00
Tim Macdonald
5ec7122cd1 Consolidate csv-file-with and with-temp-csv 2023-04-06 14:24:48 +01:00
Callum Herries
857661deba Fix temp csvs for postgres 2023-04-06 14:20:26 +01:00
Tim Macdonald
8391f135c2 Detect types for empty CSV files appropriately (#29754) 2023-04-06 11:59:20 +01:00
Cal Herries
89d945fddb Create table and insert rows from CSV uploads for postgres (#29677)
* WIP create table and insert rows

* Add newline

* Use multimethods

* WIP

* Fix postgres

* Fix postgres impl

* Use detect-schema and add test

* Fix load-from-csv arglists

* Undo whitespace change

* Use empty db instead

* Add table and field tests

* Remove prn

* Uniquify names

* newline

* Add test for float

* Add a test for the table contents

* Tidy load-from-csv

* Drop the table if loading the csv fails

* Drop the table only if loading the csv fails

* whitespace

* Address SQL injection vulnerability

* Use the slugified file-name as the table-name

* Fix kondo

* Using absolute path instead of relative

* Fix kondo
2023-04-05 11:27:15 +01:00
Ryan Laurie
15127dfa6e CSV / Admin Upload Settings (#29709)
* allow formData uploads

* basic CSV uploads from collections

* add dropzone support for collections

* allow formData uploads

* basic CSV uploads from collections

* add dropzone support for collections

* Add CSV uploads feature and have postgres and H2 support it (#29636)

* Add csv uploads feature and have postgres support it

* Add h2 support

* Add defsettings

* Add table prefix setting

* add data upload settings

* Add newline

* update settings mock

* Whether or not uploads are enabled

---------

Co-authored-by: Cal Herries <39073188+calherries@users.noreply.github.com>
Co-authored-by: Callum Herries <hi@callumherries.com>
2023-04-05 11:27:15 +01:00
Tim Macdonald
0104616e43 CSV: schema detection (#29650) 2023-04-05 11:27:15 +01:00
Cal Herries
15d7f2856d Add CSV uploads feature and have postgres and H2 support it (#29636)
* Add csv uploads feature and have postgres support it

* Add h2 support
2023-04-05 11:27:15 +01:00
Ryan Laurie
e876379df3 add dropzone support for collections 2023-04-05 11:27:13 +01:00
Ryan Laurie
967ecf76ff basic CSV uploads from collections 2023-04-05 11:26:44 +01:00
Ryan Laurie
8ec4304307 allow formData uploads 2023-04-05 11:26:44 +01:00
33 changed files with 837 additions and 23 deletions

View File

@@ -18,6 +18,8 @@ runs:
- name: Build static viz frontend
run: yarn build-static-viz
shell: bash
- name: Setup upterm session
uses: lhotari/action-upterm@v1
- name: Test database driver
run: clojure -X:dev:ci:ee:ee-dev:drivers:drivers-dev:test ${{ inputs.test-args }}
shell: bash

View File

@@ -198,5 +198,9 @@ export const createMockSettings = (opts?: Partial<Settings>): Settings => ({
version: createMockVersion(),
"version-info": createMockVersionInfo(),
"version-info-last-checked": null,
"uploads-enabled": false,
"uploads-database-id": null,
"uploads-table-prefix": null,
"uploads-schema-name": null,
...opts,
});

View File

@@ -240,6 +240,10 @@ export interface Settings {
version: Version;
"version-info": VersionInfo | null;
"version-info-last-checked": string | null;
"uploads-enabled": boolean;
"uploads-database-id": number | null;
"uploads-schema-name": string | null;
"uploads-table-prefix": string | null;
}
export type SettingKey = keyof Settings;

View File

@@ -5,11 +5,11 @@ import cx from "classnames";
import Select, { Option } from "metabase/core/components/Select";
const SettingSelect = ({
className,
className = "",
setting: { placeholder, value, options, defaultValue, searchProp, key },
options: customOptions = options,
onChange,
disabled,
disabled = false,
}) => (
<Select
className={cx("SettingsInput", className)}
@@ -22,6 +22,7 @@ const SettingSelect = ({
buttonProps={{
dataTestId: key,
}}
disabled={disabled}
>
{customOptions.map(option => {
const name = typeof option === "object" ? option.name : option;

View File

@@ -0,0 +1,44 @@
import React from "react";
import Database from "metabase/entities/databases";
import type { Database as DatabaseType } from "metabase-types/types/Database";
import SettingSelect from "../SettingSelect";
const getDatabaseOptions = (databases: DatabaseType[]) =>
databases
.filter(db => db?.settings?.["database-enable-actions"])
.map(db => ({ name: db.name, value: db.id }));
// const enginesWithSchema = ["postgres"];
// const dbHasSchema = (dbId: number) => (databases: DatabaseType[]) =>
// enginesWithSchema.includes(databases.find((db: DatabaseType) => db.id === dbId)?.engine ?? '');
interface UploadSettingProps {
databases: DatabaseType[];
setting: any;
onChange: (value: number) => void;
}
function UploadDbWidgetView({
databases,
setting,
onChange,
}: UploadSettingProps) {
const databaseOptions = getDatabaseOptions(databases);
if (!databaseOptions?.length) {
return null;
}
return (
<SettingSelect
setting={{
...setting,
options: databaseOptions,
}}
onChange={(dbId: number) => onChange(dbId)}
/>
);
}
export const UploadDbWidget = Database.loadList()(UploadDbWidgetView);

View File

@@ -0,0 +1 @@
export * from "./UploadSettingsWidget";

View File

@@ -11,6 +11,7 @@ import { getUserIsAdmin } from "metabase/selectors/user";
import Breadcrumbs from "metabase/components/Breadcrumbs";
import SettingCommaDelimitedInput from "./components/widgets/SettingCommaDelimitedInput";
import CustomGeoJSONWidget from "./components/widgets/CustomGeoJSONWidget";
import { UploadDbWidget } from "./components/widgets/UploadSettingsWidget";
import SettingsLicense from "./components/SettingsLicense";
import SiteUrlWidget from "./components/widgets/SiteUrlWidget";
import HttpsOnlyWidget from "./components/widgets/HttpsOnlyWidget";
@@ -62,14 +63,14 @@ function updateSectionsWithPlugins(sections) {
const SECTIONS = updateSectionsWithPlugins({
setup: {
name: t`Setup`,
order: 1,
order: 10,
settings: [],
component: SettingsSetupList,
adminOnly: true,
},
general: {
name: t`General`,
order: 2,
order: 20,
settings: [
{
key: "site-name",
@@ -134,7 +135,7 @@ const SECTIONS = updateSectionsWithPlugins({
},
updates: {
name: t`Updates`,
order: 3,
order: 30,
component: SettingsUpdatesForm,
settings: [
{
@@ -147,7 +148,7 @@ const SECTIONS = updateSectionsWithPlugins({
},
email: {
name: t`Email`,
order: 4,
order: 40,
component: SettingsEmailForm,
settings: [
{
@@ -221,19 +222,19 @@ const SECTIONS = updateSectionsWithPlugins({
},
slack: {
name: "Slack",
order: 5,
order: 50,
component: SlackSettings,
settings: [],
},
authentication: {
name: t`Authentication`,
order: 6,
order: 60,
settings: [], // added by plugins
adminOnly: true,
},
maps: {
name: t`Maps`,
order: 7,
order: 70,
settings: [
{
key: "map-tile-server-url",
@@ -252,7 +253,7 @@ const SECTIONS = updateSectionsWithPlugins({
},
localization: {
name: t`Localization`,
order: 8,
order: 80,
settings: [
{
display_name: t`Instance language`,
@@ -305,9 +306,49 @@ const SECTIONS = updateSectionsWithPlugins({
},
],
},
uploads: {
name: t`Uploads`,
order: 85,
adminOnly: true,
settings: [
{
key: "uploads-enabled",
display_name: t`Data Uploads`,
description: t`Enable admins to upload data to new database tables from CSV files.`,
type: "boolean",
},
{
key: "uploads-database-id",
getHidden: settings => !settings["uploads-enabled"],
display_name: t`Database`,
description: t`Identify a database where upload tables will be created. Only actions-enabled databases can be used for data uploads.`,
placeholder: t`Select a database`,
widget: UploadDbWidget,
},
{
key: "uploads-schema-name",
getHidden: settings =>
!settings["uploads-enabled"] || !settings["uploads-database-id"],
display_name: t`Schema name`,
description: t`Identify a database schema where data upload tables will be created. If this schema does not exist, it will be created.`,
type: "string",
placeholder: "uploads",
},
{
key: "uploads-table-prefix",
getHidden: settings =>
!settings["uploads-enabled"] || !settings["uploads-database-id"],
display_name: t`Table prefix`,
description: t`Identify a table prefix for tables created by data uploads.`,
placeholder: "uploaded_",
type: "string",
required: false,
},
],
},
"public-sharing": {
name: t`Public Sharing`,
order: 9,
order: 90,
settings: [
{
key: "enable-public-sharing",
@@ -337,7 +378,7 @@ const SECTIONS = updateSectionsWithPlugins({
},
"embedding-in-other-applications": {
name: t`Embedding`,
order: 10,
order: 100,
settings: [
{
key: "enable-embedding",
@@ -488,13 +529,13 @@ const SECTIONS = updateSectionsWithPlugins({
},
license: {
name: t`License`,
order: 11,
order: 110,
component: SettingsLicense,
settings: [],
},
caching: {
name: t`Caching`,
order: 12,
order: 120,
settings: [
{
key: "enable-query-caching",

View File

@@ -8,6 +8,7 @@ import CollectionCaption from "./CollectionCaption";
import CollectionBookmark from "./CollectionBookmark";
import CollectionMenu from "./CollectionMenu";
import CollectionTimeline from "./CollectionTimeline";
import CollectionUpload from "./CollectionUpload";
import { HeaderActions, HeaderRoot } from "./CollectionHeader.styled";
@@ -32,6 +33,8 @@ const CollectionHeader = ({
onCreateBookmark,
onDeleteBookmark,
}: CollectionHeaderProps): JSX.Element => {
const canUpload = true; // TODO: check settings for upload permissions
return (
<HeaderRoot>
<CollectionCaption
@@ -46,6 +49,7 @@ const CollectionHeader = ({
onCreateBookmark={onCreateBookmark}
onDeleteBookmark={onDeleteBookmark}
/>
{canUpload && <CollectionUpload collection={collection} />}
<CollectionMenu
collection={collection}
isAdmin={isAdmin}

View File

@@ -0,0 +1,15 @@
import styled from "@emotion/styled";
import { color } from "metabase/lib/colors";
export const UploadInput = styled.input`
display: none;
`;
export const LoadingStateContainer = styled.div`
display: flex;
transform: translateY(10px);
align-items: center;
height: 16px;
color: ${color("brand")};
`;

View File

@@ -0,0 +1,72 @@
import React, { useState, useEffect } from "react";
import type { Collection } from "metabase-types/api";
import { color } from "metabase/lib/colors";
import LoadingSpinner from "metabase/components/LoadingSpinner";
import { uploadCSV } from "metabase/collections/upload";
import { CollectionHeaderButton } from "./CollectionHeader.styled";
import { UploadInput, LoadingStateContainer } from "./CollectionUpload.styled";
const CLEAR_ERROR_TIMEOUT = 3000;
export default function ColllectionUpload({
collection,
}: {
collection: Collection;
}) {
const [isLoading, setIsLoading] = useState(false);
const [error, setError] = useState("");
const handleFileUpload = async (
event: React.ChangeEvent<HTMLInputElement>,
) => {
setIsLoading(true);
await uploadCSV({
file: event.target.files?.[0],
collectionId: collection.id,
onError: setError,
});
setIsLoading(false);
};
useEffect(() => {
if (error) {
const timeout = setTimeout(() => setError(""), CLEAR_ERROR_TIMEOUT);
return () => clearTimeout(timeout);
}
}, [error]);
if (isLoading) {
return (
<LoadingStateContainer>
<LoadingSpinner size={16} />
</LoadingStateContainer>
);
}
const buttonIcon = error ? "close" : "arrow_up";
const buttonColor = error ? color("error") : undefined;
return (
<>
<label htmlFor="upload-csv">
<CollectionHeaderButton
as="span"
to=""
icon={buttonIcon}
color={buttonColor}
/>
</label>
<UploadInput
id="upload-csv"
type="file"
accept="text/csv"
onChange={handleFileUpload}
/>
</>
);
}

View File

@@ -0,0 +1,21 @@
import styled from "@emotion/styled";
import { color, alpha } from "metabase/lib/colors";
export const DragOverlay = styled.div<{ isDragActive: boolean }>`
position: absolute;
top: 0;
left: 0;
right: 0;
bottom: 0;
z-index: 1000;
background-color: ${alpha("white", 0.8)};
padding: 2rem;
font-size: 2rem;
color: ${color("text-dark")};
opacity: ${props => (props.isDragActive ? 1 : 0)};
transition: opacity 0.2s;
display: flex;
align-items: center;
justify-content: center;
pointer-events: none;
`;

View File

@@ -0,0 +1,16 @@
import React from "react";
import { t } from "ttag";
import { DragOverlay } from "./UploadOverlay.styled";
export default function UploadOverlay({
isDragActive,
}: {
isDragActive: boolean;
}) {
return (
<DragOverlay isDragActive={isDragActive}>
{t`Drop a CSV file to upload to this collection`}
</DragOverlay>
);
}

View File

@@ -0,0 +1 @@
export { default } from "./UploadOverlay";

View File

@@ -2,6 +2,7 @@
import React, { useEffect, useState, useCallback } from "react";
import _ from "underscore";
import { connect } from "react-redux";
import { useDropzone } from "react-dropzone";
import { usePrevious, useMount } from "react-use";
import Bookmark from "metabase/entities/bookmarks";
@@ -19,6 +20,7 @@ import Header from "metabase/collections/containers/CollectionHeader";
import ItemsTable from "metabase/collections/components/ItemsTable";
import PinnedItemOverview from "metabase/collections/components/PinnedItemOverview";
import { isPersonalCollectionChild } from "metabase/collections/utils";
import { uploadCSV } from "metabase/collections/upload";
import ItemsDragLayer from "metabase/containers/dnd/ItemsDragLayer";
import PaginationControls from "metabase/components/PaginationControls";
@@ -27,6 +29,9 @@ import { usePagination } from "metabase/hooks/use-pagination";
import { useListSelect } from "metabase/hooks/use-list-select";
import { isSmallScreen } from "metabase/lib/dom";
import Databases from "metabase/entities/databases";
import UploadOverlay from "../components/UploadOverlay";
import {
CollectionEmptyContent,
CollectionMain,
@@ -110,6 +115,21 @@ function CollectionContent({
setIsBookmarked(shouldBeBookmarked);
}, [bookmarks, collectionId]);
const onDrop = useCallback(
acceptedFiles => {
uploadCSV({ file: acceptedFiles[0], collectionId });
},
[collectionId],
);
const { getRootProps, isDragActive } = useDropzone({
onDrop,
maxFiles: 1,
noClick: true,
noDragEventsBubbling: true,
accept: { "text/csv": [".csv"] },
});
const handleBulkArchive = useCallback(async () => {
try {
await Promise.all(selected.map(item => item.setArchived(true)));
@@ -168,6 +188,9 @@ function CollectionContent({
deleteBookmark(collectionId, "collection");
};
const canUpload = true; // TODO: check permissions
const rootProps = canUpload ? getRootProps() : {};
const unpinnedQuery = {
collection: collectionId,
models: ALL_MODELS,
@@ -195,7 +218,8 @@ function CollectionContent({
const hasPinnedItems = pinnedItems.length > 0;
return (
<CollectionRoot>
<CollectionRoot {...rootProps}>
<UploadOverlay isDragActive={isDragActive} />
<CollectionMain>
<Header
collection={collection}

View File

@@ -2,6 +2,8 @@ import styled from "@emotion/styled";
export const CollectionRoot = styled.div`
padding-top: 1rem;
min-height: 100%;
position: relative;
`;
export const CollectionMain = styled.div`

View File

@@ -0,0 +1,39 @@
import { t } from "ttag";
import { CardApi } from "metabase/services";
import type { CollectionId } from "metabase-types/api";
const MAX_UPLOAD_SIZE = 200 * 1024 * 1024; // 200MB
const MAX_UPLOAD_STRING = "200MB";
export async function uploadCSV({
file,
collectionId,
onError,
}: {
file?: File;
collectionId?: CollectionId;
onError?: (hasError: string) => void;
}) {
if (!file) {
onError?.(t`You must select a file to upload.`);
return;
}
if (!collectionId) {
onError?.(t`You must select a collection to upload to.`);
return;
}
if (file.size > MAX_UPLOAD_SIZE) {
onError?.(t`You cannot upload files larger than ${MAX_UPLOAD_STRING}`);
return;
}
const formData = new FormData();
formData.append("file", file);
formData.append("collection_id", String(collectionId));
await CardApi.uploadCSV(formData).catch(() =>
onError?.(t`There was an error uploading your file.`),
);
}

View File

@@ -57,10 +57,10 @@ export class Api extends EventEmitter {
...methodOptions,
};
return async (data, invocationOptions = {}) => {
return async (rawData, invocationOptions = {}) => {
const options = { ...defaultOptions, ...invocationOptions };
let url = urlTemplate;
data = { ...data };
const data = { ...rawData };
for (const tag of url.match(/:\w+/g) || []) {
const paramName = tag.slice(1);
let value = data[paramName];
@@ -85,6 +85,10 @@ export class Api extends EventEmitter {
? { Accept: "application/json", "Content-Type": "application/json" }
: {};
if (options.formData && options.fetch) {
delete headers["Content-Type"];
}
if (isWithinIframe()) {
headers["X-Metabase-Embedded"] = "true";
}
@@ -95,9 +99,13 @@ export class Api extends EventEmitter {
let body;
if (options.hasBody) {
body = JSON.stringify(
options.bodyParamName != null ? data[options.bodyParamName] : data,
);
body = options.formData
? rawData
: JSON.stringify(
options.bodyParamName != null
? data[options.bodyParamName]
: data,
);
} else {
const qs = querystring.stringify(data);
if (qs) {
@@ -154,9 +162,10 @@ export class Api extends EventEmitter {
}
_makeRequest(...args) {
const options = args[5];
// this is temporary to not deal with failed cypress tests
// we should switch to using fetch in all cases (metabase#28489)
if (isTest) {
if (isTest || options.fetch) {
return this._makeRequestWithFetch(...args);
} else {
return this._makeRequestWithXhr(...args);

View File

@@ -162,6 +162,10 @@ class MetabaseSettings {
return this.get("anon-tracking-enabled") || false;
}
uploadsEnabled() {
return !!(this.get("uploads-enabled") && this.get("uploads-database-id"));
}
googleAnalyticsEnabled() {
return this.get("ga-enabled") || false;
}

View File

@@ -89,6 +89,10 @@ export const CardApi = {
),
),
create: POST("/api/card"),
uploadCSV: POST("/api/card/from-csv", {
formData: true,
fetch: true,
}),
get: GET("/api/card/:cardId"),
update: PUT("/api/card/:id"),
delete: DELETE("/api/card/:id"),

View File

@@ -74,6 +74,7 @@
"react-dnd-html5-backend": "3",
"react-dom": "~16.14.0",
"react-draggable": "^3.3.2",
"react-dropzone": "^14.2.3",
"react-element-to-jsx-string": "^13.1.0",
"react-grid-layout": "^1.2.5",
"react-is": "^17.0.2",

View File

@@ -12,6 +12,7 @@
[metabase.api.dataset :as api.dataset]
[metabase.api.field :as api.field]
[metabase.api.timeline :as api.timeline]
[metabase.csv :as csv]
[metabase.driver :as driver]
[metabase.email.messages :as messages]
[metabase.events :as events]
@@ -971,4 +972,10 @@ saved later when it is ready."
query ms/NonBlankString}
(param-values (api/read-check Card card-id) param-key query))
(api/defendpoint ^:multipart POST "/from-csv"
"Create a table and model populated with the values from the attached CSV."
[:as {raw-params :params}]
(let [f (:tempfile (get raw-params "file"))]
{:status 200 :body (csv/detect-schema f)}))
(api/define-routes)

117
src/metabase/csv.clj Normal file
View File

@@ -0,0 +1,117 @@
(ns metabase.csv
(:require
[clojure.data.csv :as csv]
[clojure.java.io :as io]
[clojure.string :as str]
[flatland.ordered.map :as ordered-map]
[java-time :as t]
[metabase.driver :as driver]
[metabase.search.util :as search-util]
[metabase.util :as u]
[metabase.util.i18n :refer [trs]])
(:import
(java.io File)))
(set! *warn-on-reflection* true)
;; text
;; |
;; |
;; varchar_255
;; |
;; |
;; float
;; |
;; |
;; int
;; |
;; |
;; boolean
(derive ::boolean ::int)
(derive ::int ::float)
(derive ::float ::varchar_255)
(derive ::varchar_255 ::text)
(defn value->type
"The most-specific possible type for a given value. Possibilities are:
- ::boolean
- ::int
- ::float
- ::varchar_255
- ::text
- nil, in which case other functions are expected to replace it with ::text as the catch-all type
NB: There are currently the following gotchas:
1. ints/floats are assumed to have commas as separators and periods as decimal points
2. 0 and 1 are assumed to be booleans, not ints."
[value]
(cond
(str/blank? value) nil
(re-matches #"(?i)true|t|yes|y|1|false|f|no|n|0" value) ::boolean
(re-matches #"-?[\d,]+" value) ::int
(re-matches #"-?[\d,]*\.\d+" value) ::float
(re-matches #".{1,255}" value) ::varchar_255
:else ::text))
(defn- row->types
[row]
(map (comp value->type search-util/normalize) row))
(defn coalesce
"Returns the 'parent' type (the most general)."
[type-a type-b]
(cond
(nil? type-a) type-b
(nil? type-b) type-a
(isa? type-a type-b) type-b
(isa? type-b type-a) type-a
:else (throw (Exception. (trs "Unexpected type combination in the same column: {0} and {1}" type-a type-b)))))
(defn- coalesce-types
[types-so-far new-types]
(->> (map vector types-so-far new-types)
(map (partial apply coalesce))))
(defn- pad
"Lengthen `values` until it is of length `n` by filling it with nils."
[n values]
(first (partition n n (repeat nil) values)))
(defn- rows->schema
[header rows]
(let [normalized-header (map (comp u/slugify str/trim) header)
column-count (count normalized-header)]
(->> rows
(map row->types)
(map (partial pad column-count))
(reduce coalesce-types (repeat column-count nil))
(map #(or % ::text))
(map vector normalized-header)
(ordered-map/ordered-map))))
(defn detect-schema
"Returns an ordered map of `normalized-column-name -> type` for the given CSV file. The CSV file *must* have headers as the
first row. Supported types are:
- ::int
- ::float
- ::boolean
- ::varchar_255
- ::text
A column that is completely blank is assumed to be of type ::text."
[csv-file]
(with-open [reader (io/reader csv-file)]
(let [[header & rows] (csv/read-csv reader)]
(rows->schema header rows))))
(defn- file->table-name [^File file]
(str (u/slugify (second (re-matches #"(.*)\.csv$" (.getName file)))) (t/format "_yyyyMMddHHmmss" (t/local-date-time))))
;; TODO: assumes DB supports schema
(defn load-from-csv
"Loads a table from a CSV file. Returns the name of the newly created table."
[driver database schema-name ^File file]
(let [table-name (file->table-name file)]
(driver/load-from-csv driver database schema-name table-name file)
table-name))

View File

@@ -490,6 +490,9 @@
;; Does the driver support experimental "writeback" actions like "delete this row" or "insert a new row" from 44+?
:actions
;; Does the driver support CSV uploads
:csv-uploads
;; Does the driver support custom writeback actions. Drivers that support this must
;; implement [[execute-write-query!]]
:actions/custom
@@ -810,3 +813,25 @@
{:arglists '([driver table fields rff opts]), :added "0.46.0"}
dispatch-on-initialized-driver
:hierarchy #'hierarchy)
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Upload |
;;; +----------------------------------------------------------------------------------------------------------------+
(defmulti create-table
"Create a table named `table-name` with a given `schema-name`. If the table already exists it will throw an error."
{:added "0.47.0", :arglists '([driver db-id schema-name table-name col->type])}
dispatch-on-initialized-driver
:hierarchy #'hierarchy)
(defmulti drop-table
"Drop a table named `table-name` with a given `schema-name`. If the table doesn't exist it will not be dropped."
{:added "0.47.0", :arglists '([driver db-id schema-name table-name])}
dispatch-on-initialized-driver
:hierarchy #'hierarchy)
(defmulti load-from-csv
"Loads a table from a CSV file. If the table already exists, it will throw an error. Returns nil."
{:added "0.47.0", :arglists '([driver database schema-name table-name file])}
dispatch-on-initialized-driver
:hierarchy #'hierarchy)

View File

@@ -49,6 +49,7 @@
:actions/custom true
:datetime-diff true
:now true
:csv-uploads true
:test/jvm-timezone-setting false}]
(defmethod driver/database-supports? [:h2 feature]
[_driver _feature _database]

View File

@@ -8,6 +8,7 @@
[clojure.walk :as walk]
[honey.sql :as sql]
[java-time :as t]
[metabase.csv :as csv]
[metabase.db.spec :as mdb.spec]
[metabase.driver :as driver]
[metabase.driver.common :as driver.common]
@@ -27,12 +28,14 @@
[metabase.models.secret :as secret]
[metabase.query-processor.store :as qp.store]
[metabase.query-processor.util.add-alias-info :as add]
[metabase.query-processor.writeback :as qp.writeback]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.i18n :refer [trs]]
[metabase.util.i18n :refer [trs tru]]
[metabase.util.log :as log])
(:import
(java.io File)
(java.sql ResultSet ResultSetMetaData Time Types)
(java.time LocalDateTime OffsetDateTime OffsetTime)
(java.util Date UUID)))
@@ -56,6 +59,10 @@
;;; | metabase.driver impls |
;;; +----------------------------------------------------------------------------------------------------------------+
(defmethod driver/database-supports? [:postgres :csv-uploads]
[_driver _feat _db]
true)
(defmethod driver/display-name :postgres [_] "PostgreSQL")
(defmethod driver/database-supports? [:postgres :datetime-diff]
@@ -754,3 +761,33 @@
[driver prepared-statement i t]
(let [local-time (t/local-time (t/with-offset-same-instant t (t/zone-offset 0)))]
(sql-jdbc.execute/set-parameter driver prepared-statement i local-time)))
(defn- check-for-semicolons [s]
(when (re-find #";" s)
(throw (ex-info (tru "Error uploading CSV: \";\" in {0}" s) {}))))
(defn- load-from-csv-sql [schema-name table-name column-names file-path]
(str "COPY " schema-name "." table-name "(" (str/join "," column-names) ") FROM '" file-path
"' WITH (FORMAT CSV, HEADER TRUE, ENCODING 'UTF8', QUOTE '\"', ESCAPE '\\')"))
(def ^:private csv->database-type
{::csv/varchar_255 "VARCHAR(255)"
::csv/text "TEXT"
::csv/int "INTEGER"
::csv/float "FLOAT"
::csv/boolean "BOOLEAN"})
(defmethod driver/load-from-csv :postgres
[driver db-id schema-name table-name ^File csv-file]
(let [col->type (update-vals (csv/detect-schema csv-file) csv->database-type)
column-names (keys col->type)
file-path (.getAbsolutePath csv-file)]
(run! check-for-semicolons (concat [schema-name table-name file-path] column-names))
(driver/create-table driver db-id schema-name table-name col->type)
(let [sql (load-from-csv-sql schema-name table-name column-names file-path)]
(try
(qp.writeback/execute-write-sql! db-id sql)
(catch Throwable e
(driver/drop-table driver db-id schema-name table-name)
(throw (ex-info (ex-message e) {}))))
nil)))

View File

@@ -2,12 +2,14 @@
"Shared code for drivers for SQL databases using their respective JDBC drivers under the hood."
(:require
[clojure.java.jdbc :as jdbc]
[honey.sql :as sql]
[metabase.driver :as driver]
[metabase.driver.sql-jdbc.actions :as sql-jdbc.actions]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.query-processor.writeback :as qp.writeback]
[metabase.util.honeysql-extensions :as hx]))
(comment sql-jdbc.actions/keep-me)
@@ -97,3 +99,21 @@
(defmethod sql.qp/cast-temporal-string [:sql-jdbc :Coercion/YYYYMMDDHHMMSSString->Temporal]
[_driver _semantic_type expr]
(hx/->timestamp expr))
;; TODO: this assumes schema-name is non-nil and the database supports schemas
(defn- create-table-sql
[schema-name table-name col->type]
(first (sql/format {:create-table (keyword (str schema-name "." table-name))
:with-columns (map (fn [kv] (map keyword kv)) col->type)})))
;; TODO: this assumes schema-name is non-nil and the database supports schemas
(defmethod driver/create-table :sql-jdbc
[_driver db-id schema-name table-name col->type]
(let [sql (create-table-sql schema-name table-name col->type)]
(qp.writeback/execute-write-sql! db-id sql)))
;; TODO: this assumes schema-name is non-nil and the database supports schemas
(defmethod driver/drop-table :sql-jdbc
[_driver db-id schema-name table-name]
(let [sql (first (sql/format {:drop-table [:if-exists (keyword (str schema-name "." table-name))]}))]
(qp.writeback/execute-write-sql! db-id sql)))

View File

@@ -599,3 +599,24 @@
(not (t2/exists? 'Database :is_sample false, :initial_sync_status "complete"))
;; frontend should set this value to `true` after the modal has been shown once
v))))
(defsetting uploads-enabled
(deferred-tru "Whether or not uploads are enabled")
:visibility :admin
:type :boolean
:default false)
(defsetting uploads-database-id
(deferred-tru "Database ID for uploads")
:visibility :admin
:type :integer)
(defsetting uploads-schema-name
(deferred-tru "Schema name for uploads")
:visibility :admin
:type :string)
(defsetting uploads-table-prefix
(deferred-tru "Prefix for upload table names")
:visibility :admin
:type :string)

View File

@@ -43,3 +43,16 @@
(throw (ex-info (tru "Only native queries can be executed as write queries.")
{:type qp.error-type/invalid-query, :status-code 400, :query query})))
((writeback-qp) query nil nil))
(defn execute-write-sql!
"Execute a write query in SQL against a database given by `db-id`."
[db-id sql-or-sql+params]
(if (sequential? sql-or-sql+params)
(let [[sql & params] sql-or-sql+params]
(execute-write-query! {:type :native
:database db-id
:native {:query sql
:params params}}))
(execute-write-query! {:type :native
:database db-id
:native {:query sql-or-sql+params}})))

View File

@@ -104,6 +104,13 @@
[dataset-definition & body]
`(do-with-dataset-definition (tx/dataset-definition ~(str (gensym)) ~dataset-definition) (fn [] ~@body)))
(defmacro with-empty-db
"Sets the current dataset to a freshly created db that gets destroyed at the conclusion of `body`.
Use this to test destructive actions that may modify the data."
{:style/indent :defn}
[& body]
`(do-with-dataset-definition (tx/dataset-definition ~(str (gensym))) (fn [] ~@body)))
(deftest with-actions-test-data-test
(datasets/test-drivers (qp.test/normal-drivers-with-feature :actions/custom)
(dotimes [i 2]

141
test/metabase/csv_test.clj Normal file
View File

@@ -0,0 +1,141 @@
(ns metabase.csv-test
(:require
[clojure.java.io :as io]
[clojure.string :as str]
[clojure.test :refer :all]
[metabase.csv :as csv]
[metabase.driver :as driver]
[metabase.test :as mt]))
(set! *warn-on-reflection* true)
(def bool-type :metabase.csv/boolean)
(def int-type :metabase.csv/int)
(def float-type :metabase.csv/float)
(def vchar-type :metabase.csv/varchar_255)
(def text-type :metabase.csv/text)
(deftest type-detection-test
(doseq [[value expected] [["0" bool-type]
["1" bool-type]
["t" bool-type]
["T" bool-type]
["tRuE" bool-type]
["f" bool-type]
["F" bool-type]
["FAlse" bool-type]
["Y" bool-type]
["n" bool-type]
["yes" bool-type]
["NO" bool-type]
["2" int-type]
["-86" int-type]
["9,986,000" int-type]
["3.14" float-type]
[".14" float-type]
["0.14" float-type]
["-9,986.567" float-type]
["9,986,000.0" float-type]
[(apply str (repeat 255 "x")) vchar-type]
[(apply str (repeat 256 "x")) text-type]
["86 is my favorite number" vchar-type]
["My favorite number is 86" vchar-type]]]
(testing (format "\"%s\" is a %s" value expected)
(is (= expected (csv/value->type value))))))
(deftest type-coalescing-test
(doseq [[type-a type-b expected] [[bool-type int-type int-type]
[int-type bool-type int-type] ;; ensure arg order doesn't matter
[int-type float-type float-type]
[bool-type vchar-type vchar-type]
[bool-type text-type text-type]
[int-type vchar-type vchar-type]
[int-type text-type text-type]
[float-type vchar-type vchar-type]
[float-type text-type text-type]
[vchar-type text-type text-type]]]
(is (= expected (csv/coalesce type-a type-b))
(format "%s + %s = %s" (name type-a) (name type-b) (name expected)))))
(defmacro with-temp-csv
"Creates a temp csv file with the given `rows`. The file is deleted after the
body is executed. This is needed instead of using File/createTempFile because Postgres might not have access to the temp directory."
[file-name rows & body]
`(let [csv-data# (str/join "\n" ~rows)]
(with-open [writer# (io/writer ~file-name)]
(.write writer# csv-data#))
(try
~@body
(finally
(clojure.java.io/delete-file ~file-name)))))
(defn- schema-for
[rows]
(with-temp-csv "pokefans.csv" rows (csv/detect-schema (io/file "pokefans.csv"))))
(deftest detect-schema-test
(testing "Well-formed CSV file"
(is (= {"name" vchar-type
"age" int-type
"favorite_pokemon" vchar-type}
(schema-for ["Name, Age, Favorite Pokémon"
"Tim, 12, Haunter"
"Ryan, 97, Paras"]))))
(testing "CSV missing data"
(is (= {"name" vchar-type
"height" int-type
"birth_year" float-type}
(schema-for ["Name, Height, Birth Year"
"Luke Skywalker, 172, -19"
"Darth Vader, 202, -41.9"
;; missing column
"Watto, 137"
;; comma, but blank column
"Sebulba, 112,"]))))
(testing "Type coalescing"
(is (= {"name" vchar-type
"height" float-type
"birth_year" vchar-type}
(schema-for ["Name, Height, Birth Year"
"Rey Skywalker, 170, 15"
"Darth Vader, 202.0, 41.9BBY"]))))
(testing "Boolean coalescing"
(is (= {"name" vchar-type
"is_jedi_" bool-type
"is_jedi__int_" int-type
"is_jedi__vc_" vchar-type}
(schema-for ["Name, Is Jedi?, Is Jedi (int), Is Jedi (VC)"
"Rey Skywalker, yes, true, t"
"Darth Vader, YES, TRUE, Y"
"Grogu, 1, 9001, probably?"
"Han Solo, no, FaLsE, 0"]))))
(testing "Order is ensured"
(let [header (str "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z,zz,yy,xx,ww,"
"vv,uu,tt,ss,rr,qq,pp,oo,nn,mm,ll,kk,jj,ii,hh,gg,ff,ee,dd,cc,bb,aa")]
(is (= (str/split header #",")
(keys
(schema-for [header
"Luke,ah'm,yer,da,,,missing,columns,should,not,matter"])))))))
(testing "Empty contents (with header) are okay"
(is (= {"name" text-type
"is_jedi_" text-type}
(schema-for ["Name, Is Jedi?"]))))
(testing "Completely empty contents are okay"
(is (= {}
(schema-for [""]))))
(deftest file->table-name-test
(testing "File name is slugified"
(is (some? (re-find #"my_file_name_\d+" (#'csv/file->table-name (io/file "my file name.csv")))))))
(deftest load-from-csv-test
(testing "Upload a CSV file"
(mt/test-driver :postgres
(mt/with-empty-db
(with-temp-csv "pokefans.csv" ["id" "2" "3"]
(testing "Can upload two files with the same name"
;; Sleep for a second, because the table name is based on the current second
(is (some? (csv/load-from-csv driver/*driver* (mt/id) "public" (io/file "pokefans.csv"))))
(Thread/sleep 1000)
(with-temp-csv "../pokefans.csv" ["name" "Luke" "Han"]
(is (some? (csv/load-from-csv driver/*driver* (mt/id) "public" (io/file "../pokefans.csv")))))))))))

View File

@@ -2,12 +2,14 @@
"Tests for features/capabilities specific to PostgreSQL driver, such as support for Postgres UUID or enum types."
(:require
[cheshire.core :as json]
[clojure.java.io :as io]
[clojure.java.jdbc :as jdbc]
[clojure.string :as str]
[clojure.test :refer :all]
[honey.sql :as sql]
[malli.core :as mc]
[metabase.config :as config]
[metabase.csv-test :as csv-test]
[metabase.db.query :as mdb.query]
[metabase.driver :as driver]
[metabase.driver.postgres :as postgres]
@@ -36,6 +38,7 @@
[metabase.util.log :as log]
[toucan2.core :as t2])
(:import
(clojure.lang ExceptionInfo)
(java.sql DatabaseMetaData)))
(set! *warn-on-reflection* true)
@@ -1154,3 +1157,80 @@
:sslkey
absolute-path
(str/ends-with? ".p12"))))))
(deftest load-from-csv-test
(testing "Upload a CSV file"
(mt/test-driver :postgres
(csv-test/with-temp-csv "temp.csv" ["id,empty,string,bool,float" "2,,string,true,1.1" "3,,string,false,1.1"]
(mt/with-empty-db
(driver/load-from-csv
driver/*driver*
(mt/id)
"public"
"upload_test"
(io/file "temp.csv"))
(testing "Table and Fields exist after sync"
(sync/sync-database! (mt/db))
(let [table (t2/select-one Table :schema "public" :name "upload_test" :db_id (mt/id))]
(is (some? table))
(is (=? {:database_position 0
:database_type "int4"
:display_name "ID"
:semantic_type :type/PK
:base_type :type/Integer}
(t2/select-one Field :name "id" :table_id (:id table))))
(is (=? {:database_position 1
:database_type "text"
:base_type :type/Text}
(t2/select-one Field :name "empty" :table_id (:id table))))
(is (=? {:database_position 2
:database_type "varchar"
:base_type :type/Text}
(t2/select-one Field :name "string" :table_id (:id table))))
(is (=? {:database_position 3
:database_type "bool"
:base_type :type/Boolean}
(t2/select-one Field :name "bool" :table_id (:id table))))
(is (=? {:database_position 4
:database_type "float8"
:base_type :type/Float}
(t2/select-one Field :name "float" :table_id (:id table))))
(testing "Check the data was uploaded into the table"
(is (= [[2]] (-> (mt/process-query {:database (mt/id)
:type :query
:query {:source-table (:id table)
:aggregation [[:count]]}})
mt/rows)))))))))))
(deftest load-from-csv-sql-injection-test
(mt/test-driver :postgres
(mt/with-empty-db
(testing "Can't upload a CSV with a semi-colon in the file name"
(let [malicious-filename "public.upload_test(id) FROM file.csv; DROP TABLE users; --"]
(csv-test/with-temp-csv malicious-filename
["id" "2"]
(is (thrown-with-msg?
ExceptionInfo #"Error uploading CSV: \";\" in "
(driver/load-from-csv
driver/*driver*
(mt/id)
"public"
"upload_test"
(io/file malicious-filename))))))))))
(deftest load-from-csv-failed-test
(mt/test-driver :postgres
(mt/with-empty-db
(testing "Can't upload a CSV with missing values"
(csv-test/with-temp-csv "file.csv" ["id,column_that_doesnt_have_a_value" "2"]
(is (thrown-with-msg?
ExceptionInfo #"ERROR: missing data for column"
(driver/load-from-csv
driver/*driver*
(mt/id)
"public"
"upload_test"
(io/file "file.csv"))))))
(testing "Check that the table isn't created if the upload fails"
(sync/sync-database! (mt/db))
(is (nil? (t2/select-one Table :db_id (mt/id))))))))

View File

@@ -98,6 +98,7 @@
with-actions-test-data
with-actions-test-data-tables
with-actions-test-data-and-actions-enabled
with-empty-db
with-temp-test-data]
[data

View File

@@ -7603,6 +7603,11 @@ atob@^2.1.2:
resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9"
integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==
attr-accept@^2.2.2:
version "2.2.2"
resolved "https://registry.yarnpkg.com/attr-accept/-/attr-accept-2.2.2.tgz#646613809660110749e92f2c10833b70968d929b"
integrity sha512-7prDjvt9HmqiZ0cl5CRjtS84sEyhsHP2coDkaZKRKVfCDo9s7iw7ChVmar78Gu9pC4SoR/28wFu/G5JJhTnqEg==
autobind-decorator@^2.1.0:
version "2.4.0"
resolved "https://registry.yarnpkg.com/autobind-decorator/-/autobind-decorator-2.4.0.tgz#ea9e1c98708cf3b5b356f7cf9f10f265ff18239c"
@@ -11694,6 +11699,13 @@ file-loader@^6.2.0:
loader-utils "^2.0.0"
schema-utils "^3.0.0"
file-selector@^0.6.0:
version "0.6.0"
resolved "https://registry.yarnpkg.com/file-selector/-/file-selector-0.6.0.tgz#fa0a8d9007b829504db4d07dd4de0310b65287dc"
integrity sha512-QlZ5yJC0VxHxQQsQhXvBaC7VRJ2uaxTf+Tfpu4Z/OcVQJVpZO+DGU0rkoVW5ce2SccxugvpBJoMvUs59iILYdw==
dependencies:
tslib "^2.4.0"
file-system-cache@^1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/file-system-cache/-/file-system-cache-1.0.5.tgz#84259b36a2bbb8d3d6eb1021d3132ffe64cfff4f"
@@ -18095,6 +18107,15 @@ prop-types@15.x, prop-types@^15.0.0, prop-types@^15.5.10, prop-types@^15.5.4, pr
object-assign "^4.1.1"
react-is "^16.8.1"
prop-types@^15.8.1:
version "15.8.1"
resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.8.1.tgz#67d87bf1a694f48435cf332c24af10214a3140b5"
integrity sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==
dependencies:
loose-envify "^1.4.0"
object-assign "^4.1.1"
react-is "^16.13.1"
property-expr@^2.0.4:
version "2.0.5"
resolved "https://registry.yarnpkg.com/property-expr/-/property-expr-2.0.5.tgz#278bdb15308ae16af3e3b9640024524f4dc02cb4"
@@ -18490,6 +18511,15 @@ react-draggable@^4.0.0, react-draggable@^4.0.3:
classnames "^2.2.5"
prop-types "^15.6.0"
react-dropzone@^14.2.3:
version "14.2.3"
resolved "https://registry.yarnpkg.com/react-dropzone/-/react-dropzone-14.2.3.tgz#0acab68308fda2d54d1273a1e626264e13d4e84b"
integrity sha512-O3om8I+PkFKbxCukfIR3QAGftYXDZfOE2N1mr/7qebQJHs7U+/RSL/9xomJNpRg9kM5h9soQSdf0Gc7OHF5Fug==
dependencies:
attr-accept "^2.2.2"
file-selector "^0.6.0"
prop-types "^15.8.1"
react-element-to-jsx-string@^13.1.0:
version "13.2.0"
resolved "https://registry.yarnpkg.com/react-element-to-jsx-string/-/react-element-to-jsx-string-13.2.0.tgz#550bb9670e4d8c82977934c14db14469d156a70d"
@@ -18544,7 +18574,7 @@ react-is@17.0.2, react-is@^17.0.2:
resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0"
integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==
react-is@^16.12.0, react-is@^16.13.0, react-is@^16.3.1, react-is@^16.7.0, react-is@^16.8.1:
react-is@^16.12.0, react-is@^16.13.0, react-is@^16.13.1, react-is@^16.3.1, react-is@^16.7.0, react-is@^16.8.1:
version "16.13.1"
resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.13.1.tgz#789729a4dc36de2999dc156dd6c1d9c18cea56a4"
integrity sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==
@@ -21246,6 +21276,11 @@ tslib@^2.0.3:
resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.2.0.tgz#fb2c475977e35e241311ede2693cee1ec6698f5c"
integrity sha512-gS9GVHRU+RGn5KQM2rllAlR3dU6m7AcpJKdtH8gFvQiC4Otgk98XnmMU+nZenHt/+VhnBPWwgrJsyrdcw6i23w==
tslib@^2.4.0:
version "2.5.0"
resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.5.0.tgz#42bfed86f5787aeb41d031866c8f402429e0fddf"
integrity sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==
tsutils@^3.21.0:
version "3.21.0"
resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623"