should be it
This commit is contained in:
70
external/duckdb/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test_slow
vendored
Normal file
70
external/duckdb/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test_slow
vendored
Normal file
@@ -0,0 +1,70 @@
|
||||
# name: test/sql/copy/csv/rejects/csv_buffer_size_rejects.test_slow
|
||||
# description: Force CSV Lines from errors to fall mid-buffers
|
||||
# group: [rejects]
|
||||
|
||||
statement maybe
|
||||
FROM read_csv('${path}/bad_date.csv')
|
||||
----
|
||||
|
||||
require skip_reload
|
||||
|
||||
# Test fails on windows because byte_position is slightly different due to \r\n instead of \n.
|
||||
require notwindows
|
||||
|
||||
loop buffer_size 7 11
|
||||
|
||||
query IIIII
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/small_bad.csv',
|
||||
buffer_size=${buffer_size},
|
||||
store_rejects = true,
|
||||
columns = {'column0':'INTEGER', 'column1':'VARCHAR'});
|
||||
----
|
||||
INTEGER VARCHAR 3 3 1
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors order by all;
|
||||
----
|
||||
0 3 9 9 1 column0 CAST C,A Error when converting column "column0". Could not convert string "C" to 'INTEGER'
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
endloop
|
||||
|
||||
loop buffer_size 5 10
|
||||
|
||||
# Ensure that we can get the schema if we reduce the sample size and ignore errors
|
||||
query IIIII
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
buffer_size=${buffer_size},
|
||||
store_rejects = true);
|
||||
----
|
||||
BIGINT VARCHAR 11044 11044 2
|
||||
|
||||
query IIIIIIIIII rowsort
|
||||
SELECT * EXCLUDE (file_id, scan_id, user_arguments) FROM reject_scans order by #1;
|
||||
----
|
||||
data/csv/error/mismatch/big_bad.csv , (empty) (empty) \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL
|
||||
data/csv/error/mismatch/big_bad2.csv , (empty) (empty) \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL
|
||||
|
||||
query IIIIIIII rowsort
|
||||
SELECT * EXCLUDE (file_id, scan_id) FROM reject_errors order by all;
|
||||
----
|
||||
2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT'
|
||||
3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT'
|
||||
4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT'
|
||||
5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT'
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
endloop
|
||||
141
external/duckdb/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test
vendored
Normal file
141
external/duckdb/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test
vendored
Normal file
@@ -0,0 +1,141 @@
|
||||
# name: test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test
|
||||
# description: Test that incorrect column amounts return correct info as rejects tables
|
||||
# group: [rejects]
|
||||
|
||||
require skip_reload
|
||||
|
||||
# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n
|
||||
require notwindows
|
||||
|
||||
statement ok
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/rejects/incorrect_columns/few_columns.csv',
|
||||
columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'},
|
||||
store_rejects=true, auto_detect=false, header = 1);
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors order by all;
|
||||
----
|
||||
0 1814 14505 14510 3 d MISSING COLUMNS 1,2,3 Expected Number of Columns: 4 Found: 3
|
||||
0 1823 14575 14576 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1
|
||||
0 1823 14575 14576 2 c MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 2
|
||||
0 1823 14575 14576 3 d MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 3
|
||||
0 2378 19009 19010 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1
|
||||
0 2378 19009 19010 2 c MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 2
|
||||
0 2378 19009 19010 3 d MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 3
|
||||
0 2762 22075 22078 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 2
|
||||
0 2762 22075 22078 3 d MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 3
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
statement ok
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/rejects/incorrect_columns/many_columns.csv',
|
||||
columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'},
|
||||
store_rejects=true, auto_detect=false, header = 1, strict_mode=True);
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors order by all;
|
||||
----
|
||||
0 1096 8761 8768 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5
|
||||
0 1096 8761 8770 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6
|
||||
0 1159 9269 9276 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5
|
||||
0 1159 9269 9278 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6
|
||||
0 1206 9649 9656 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5
|
||||
0 2769 22155 22162 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
statement ok
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/rejects/incorrect_columns/mix_columns.csv',
|
||||
columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'},
|
||||
store_rejects=true, auto_detect=false, header = 1, strict_mode=True);
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors order by all;
|
||||
----
|
||||
0 1604 12825 12826 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1
|
||||
0 1604 12825 12826 2 c MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 2
|
||||
0 1604 12825 12826 3 d MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 3
|
||||
0 1671 13355 13362 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5
|
||||
0 1671 13355 13364 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6
|
||||
0 2751 21999 22002 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 2
|
||||
0 2751 21999 22002 3 d MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 3
|
||||
0 2768 22131 22138 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5
|
||||
0 2768 22131 22140 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6
|
||||
|
||||
|
||||
# Different Buffer Sizes
|
||||
loop buffer_size 10 15
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
statement ok
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/rejects/incorrect_columns/small_mix.csv',
|
||||
columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'},
|
||||
store_rejects=true, auto_detect=false, header = 1, strict_mode=True);
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors order by all
|
||||
----
|
||||
0 3 17 24 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5
|
||||
0 4 27 32 3 d MISSING COLUMNS 1,2,3 Expected Number of Columns: 4 Found: 3
|
||||
|
||||
endloop
|
||||
|
||||
# All files
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
statement ok
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/rejects/incorrect_columns/*.csv',
|
||||
columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'},
|
||||
store_rejects=true, auto_detect=false, header = 1, strict_mode=True);
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors order by all
|
||||
----
|
||||
0 1814 14505 14510 3 d MISSING COLUMNS 1,2,3 Expected Number of Columns: 4 Found: 3
|
||||
0 1823 14575 14576 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1
|
||||
0 1823 14575 14576 2 c MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 2
|
||||
0 1823 14575 14576 3 d MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 3
|
||||
0 2378 19009 19010 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1
|
||||
0 2378 19009 19010 2 c MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 2
|
||||
0 2378 19009 19010 3 d MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 3
|
||||
0 2762 22075 22078 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 2
|
||||
0 2762 22075 22078 3 d MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 3
|
||||
1 1096 8761 8768 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5
|
||||
1 1096 8761 8770 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6
|
||||
1 1159 9269 9276 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5
|
||||
1 1159 9269 9278 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6
|
||||
1 1206 9649 9656 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5
|
||||
1 2769 22155 22162 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5
|
||||
2 1604 12825 12826 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1
|
||||
2 1604 12825 12826 2 c MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 2
|
||||
2 1604 12825 12826 3 d MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 3
|
||||
2 1671 13355 13362 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5
|
||||
2 1671 13355 13364 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6
|
||||
2 2751 21999 22002 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 2
|
||||
2 2751 21999 22002 3 d MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 3
|
||||
2 2768 22131 22138 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5
|
||||
2 2768 22131 22140 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6
|
||||
3 3 17 24 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5
|
||||
3 4 27 32 3 d MISSING COLUMNS 1,2,3 Expected Number of Columns: 4 Found: 3
|
||||
70
external/duckdb/test/sql/copy/csv/rejects/csv_rejects_auto.test
vendored
Normal file
70
external/duckdb/test/sql/copy/csv/rejects/csv_rejects_auto.test
vendored
Normal file
@@ -0,0 +1,70 @@
|
||||
# name: test/sql/copy/csv/rejects/csv_rejects_auto.test
|
||||
# group: [rejects]
|
||||
|
||||
require skip_reload
|
||||
|
||||
# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n
|
||||
require notwindows
|
||||
|
||||
# Ensure that we can get the schema if we reduce the sample size and ignore errors
|
||||
query IIIII
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
store_rejects=true);
|
||||
----
|
||||
BIGINT VARCHAR 11044 11044 2
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors order by all;
|
||||
----
|
||||
0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT'
|
||||
0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT'
|
||||
1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT'
|
||||
1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT'
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
# Test with lots of errors
|
||||
query I
|
||||
SELECT SUM(num) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/half1.csv',
|
||||
header=true,
|
||||
sample_size=1,
|
||||
store_rejects=true)
|
||||
----
|
||||
2464
|
||||
|
||||
query I
|
||||
SELECT COUNT(*) FROM reject_errors;
|
||||
----
|
||||
1024
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
# Test with more errors than STANDARD_VECTOR_SIZE
|
||||
query I
|
||||
SELECT SUM(num) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/half2.csv',
|
||||
header=true,
|
||||
ignore_errors=true,
|
||||
sample_size=1,
|
||||
rejects_table='csv_rejects_table');
|
||||
----
|
||||
2542
|
||||
|
||||
query I
|
||||
SELECT COUNT(*) FROM csv_rejects_table;
|
||||
----
|
||||
3072
|
||||
|
||||
statement ok
|
||||
DROP TABLE csv_rejects_table;
|
||||
25
external/duckdb/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test
vendored
Normal file
25
external/duckdb/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test
vendored
Normal file
@@ -0,0 +1,25 @@
|
||||
# name: test/sql/copy/csv/rejects/csv_rejects_flush_cast.test
|
||||
# description: Test that Flush Cast functions properly for the rejects tables
|
||||
# group: [rejects]
|
||||
|
||||
require skip_reload
|
||||
|
||||
# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n
|
||||
require notwindows
|
||||
|
||||
query IIIII
|
||||
SELECT first(a), first(b), typeof(first(a)), typeof(first(b)), COUNT(*) FROM read_csv(
|
||||
'data/csv/error/flush_cast.csv',
|
||||
columns = {'a': 'DATE', 'b': 'VARCHAR'},
|
||||
store_rejects = true,
|
||||
delim = ',',
|
||||
dateformat = '%d-%m-%Y');
|
||||
----
|
||||
2001-09-25 bla DATE VARCHAR 2811
|
||||
|
||||
query IIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors order by all;
|
||||
----
|
||||
0 439 6997 6997 1 a CAST B, bla Error when converting column "a". Could not convert string "B" to 'DATE'
|
||||
0 2813 44972 44972 1 a CAST c, bla Error when converting column "a". Could not convert string "c" to 'DATE'
|
||||
|
||||
26
external/duckdb/test/sql/copy/csv/rejects/csv_rejects_flush_message.test
vendored
Normal file
26
external/duckdb/test/sql/copy/csv/rejects/csv_rejects_flush_message.test
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
# name: test/sql/copy/csv/rejects/csv_rejects_flush_message.test
|
||||
# description: Test that Flush Cast gives reasonable messages
|
||||
# group: [rejects]
|
||||
|
||||
require skip_reload
|
||||
|
||||
# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n
|
||||
require notwindows
|
||||
|
||||
query I
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/rejects/flush.csv',
|
||||
columns = {'a': 'DECIMAL'},
|
||||
store_rejects = true);
|
||||
----
|
||||
1521000.000
|
||||
33.439
|
||||
-21060000.000
|
||||
|
||||
query IIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors order by all;
|
||||
----
|
||||
0 2 10 10 1 a CAST "not-a-number" Error when converting column "a". Could not convert string "not-a-number" to 'DECIMAL'
|
||||
0 3 25 25 1 a CAST "-26,9568" Error when converting column "a". Could not convert string "-26,9568" to 'DECIMAL'
|
||||
0 5 44 44 1 a CAST "33,4386" Error when converting column "a". Could not convert string "33,4386" to 'DECIMAL'
|
||||
0 6 54 54 1 a CAST "33.4386,00" Error when converting column "a". Could not convert string "33.4386,00" to 'DECIMAL'
|
||||
89
external/duckdb/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test
vendored
Normal file
89
external/duckdb/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test
vendored
Normal file
@@ -0,0 +1,89 @@
|
||||
# name: test/sql/copy/csv/rejects/csv_rejects_maximum_line.test
|
||||
# description: Tests rejects tables on max line size parameter
|
||||
# group: [rejects]
|
||||
|
||||
require skip_reload
|
||||
|
||||
# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n
|
||||
require notwindows
|
||||
|
||||
statement ok
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/rejects/maximum_line/max_10.csv',
|
||||
columns = {'a': 'VARCHAR', 'b': 'INTEGER'},
|
||||
store_rejects=true, auto_detect=false, header = 1, max_line_size=10);
|
||||
|
||||
query IIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors order by all;
|
||||
----
|
||||
0 5 23 23 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 Maximum line size of 10 bytes exceeded. Actual Size:18 bytes.
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
# Test with buffer sizes
|
||||
|
||||
loop buffer_size 22 27
|
||||
|
||||
statement ok
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/rejects/maximum_line/max_10.csv',
|
||||
columns = {'a': 'VARCHAR', 'b': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=10, buffer_size=${buffer_size});
|
||||
|
||||
query IIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors order by all;
|
||||
----
|
||||
0 5 23 23 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 Maximum line size of 10 bytes exceeded. Actual Size:18 bytes.
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
endloop
|
||||
|
||||
# Test over vector size file
|
||||
statement ok
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/rejects/maximum_line/over_vector.csv',
|
||||
columns = {'a': 'VARCHAR', 'b': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=20);
|
||||
|
||||
query IIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors order by all;
|
||||
----
|
||||
0 2282 13685 13685 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 20 bytes exceeded. Actual Size:24 bytes.
|
||||
0 2591 15558 15558 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 20 bytes exceeded. Actual Size:24 bytes.
|
||||
0 2923 17569 17569 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 Maximum line size of 20 bytes exceeded. Actual Size:24 bytes.
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
# Read Multiple Files
|
||||
statement ok
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/rejects/maximum_line/*.csv',
|
||||
columns = {'a': 'VARCHAR', 'b': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=10);
|
||||
|
||||
query IIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors order by all;
|
||||
----
|
||||
0 5 23 23 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 Maximum line size of 10 bytes exceeded. Actual Size:18 bytes.
|
||||
1 2282 13685 13685 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 10 bytes exceeded. Actual Size:24 bytes.
|
||||
1 2591 15558 15558 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 10 bytes exceeded. Actual Size:24 bytes.
|
||||
1 2923 17569 17569 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 Maximum line size of 10 bytes exceeded. Actual Size:24 bytes.
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
287
external/duckdb/test/sql/copy/csv/rejects/csv_rejects_read.test
vendored
Normal file
287
external/duckdb/test/sql/copy/csv/rejects/csv_rejects_read.test
vendored
Normal file
@@ -0,0 +1,287 @@
|
||||
# name: test/sql/copy/csv/rejects/csv_rejects_read.test
|
||||
# group: [rejects]
|
||||
|
||||
require skip_reload
|
||||
|
||||
# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n
|
||||
require notwindows
|
||||
|
||||
query III rowsort
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/error/mismatch/bad.csv',
|
||||
columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'},
|
||||
store_rejects = true, auto_detect=true);
|
||||
----
|
||||
1 2 AAA
|
||||
6 7 CCC
|
||||
|
||||
query IIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors;
|
||||
----
|
||||
0 2 10 12 2 col1 CAST 4,BBB,9, Error when converting column "col1". Could not convert string "BBB" to 'INTEGER'
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
# Test with multiple columns on the same row
|
||||
query III rowsort
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/error/mismatch/bad2.csv',
|
||||
columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false);
|
||||
----
|
||||
4 5 9
|
||||
|
||||
query IIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL;
|
||||
----
|
||||
0 1 1 5 3 col2 CAST 1,2,DDD, Error when converting column "col2". Could not convert string "DDD" to 'INTEGER'
|
||||
0 3 17 17 1 col0 CAST EEE,7,FFF, Error when converting column "col0". Could not convert string "EEE" to 'INTEGER'
|
||||
0 3 17 23 3 col2 CAST EEE,7,FFF, Error when converting column "col2". Could not convert string "FFF" to 'INTEGER'
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
# Test with multiple files
|
||||
query III rowsort
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/error/mismatch/bad*.csv',
|
||||
columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'},
|
||||
store_rejects = true, auto_detect=false);
|
||||
----
|
||||
1 2 AAA
|
||||
1 2 DDD
|
||||
4 5 9
|
||||
6 7 CCC
|
||||
|
||||
query IIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL;
|
||||
----
|
||||
0 2 10 12 2 col1 CAST 4,BBB,9, Error when converting column "col1". Could not convert string "BBB" to 'INTEGER'
|
||||
1 3 17 17 1 col0 CAST EEE,7,FFF, Error when converting column "col0". Could not convert string "EEE" to 'INTEGER'
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
# Set limit
|
||||
|
||||
query III rowsort
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/error/mismatch/bad*.csv',
|
||||
columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'},
|
||||
store_rejects = true,rejects_limit=2, ignore_errors=true, auto_detect=false);
|
||||
----
|
||||
1 2 AAA
|
||||
1 2 DDD
|
||||
4 5 9
|
||||
6 7 CCC
|
||||
|
||||
# We should now only have two errors logged
|
||||
query I
|
||||
SELECT COUNT(*) FROM reject_errors
|
||||
----
|
||||
2
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
# Try with bigger files
|
||||
query I
|
||||
SELECT SUM(num) FROM read_csv(
|
||||
'data/csv/error/mismatch/big_bad.csv',
|
||||
columns = {'num': 'INTEGER', 'str': 'VARCHAR'},
|
||||
store_rejects = true, auto_detect=false);
|
||||
----
|
||||
4270
|
||||
|
||||
query IIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL;
|
||||
----
|
||||
0 2176 10876 10876 1 num CAST B, A Error when converting column "num". Could not convert string "B" to 'INTEGER'
|
||||
0 4176 20876 20876 1 num CAST C, A Error when converting column "num". Could not convert string "C" to 'INTEGER'
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query I
|
||||
SELECT SUM(num) FROM read_csv(
|
||||
'data/csv/error/mismatch/big_bad2.csv',
|
||||
columns = {'num': 'INTEGER', 'str': 'VARCHAR'},
|
||||
store_rejects = true, auto_detect=false)
|
||||
----
|
||||
6774
|
||||
|
||||
query IIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL;
|
||||
----
|
||||
0 3680 18396 18396 1 num CAST B, A Error when converting column "num". Could not convert string "B" to 'INTEGER'
|
||||
0 5680 28396 28396 1 num CAST C, A Error when converting column "num". Could not convert string "C" to 'INTEGER'
|
||||
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
# Test with multiple big files
|
||||
query I
|
||||
SELECT SUM(num) FROM read_csv(
|
||||
'data/csv/error/mismatch/big_*.csv',
|
||||
columns = {'num': 'INTEGER', 'str': 'VARCHAR'},
|
||||
store_rejects = true, auto_detect=false);
|
||||
----
|
||||
11044
|
||||
|
||||
query IIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL;
|
||||
----
|
||||
0 2176 10876 10876 1 num CAST B, A Error when converting column "num". Could not convert string "B" to 'INTEGER'
|
||||
0 4176 20876 20876 1 num CAST C, A Error when converting column "num". Could not convert string "C" to 'INTEGER'
|
||||
1 3680 18396 18396 1 num CAST B, A Error when converting column "num". Could not convert string "B" to 'INTEGER'
|
||||
1 5680 28396 28396 1 num CAST C, A Error when converting column "num". Could not convert string "C" to 'INTEGER'
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
# Test with multiple rejects table in the same query
|
||||
query IIII rowsort
|
||||
SELECT *
|
||||
FROM read_csv(
|
||||
'data/csv/error/mismatch/small1.csv',
|
||||
columns = {'num': 'INTEGER', 'str': 'VARCHAR'},
|
||||
store_rejects = true) as L
|
||||
JOIN read_csv(
|
||||
'data/csv/error/mismatch/small2.csv',
|
||||
columns = {'num': 'INTEGER', 'str': 'VARCHAR'},
|
||||
store_rejects = true) as R
|
||||
ON L.num = R.num;
|
||||
----
|
||||
1 A 1 A
|
||||
3 C 3 C
|
||||
|
||||
query IIIIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id, file_id) FROM reject_scans ORDER BY ALL;
|
||||
----
|
||||
data/csv/error/mismatch/small1.csv , (empty) (empty) \n 0 1 {'num': 'INTEGER','str': 'VARCHAR'} NULL NULL store_rejects=true
|
||||
data/csv/error/mismatch/small2.csv , (empty) (empty) \n 0 1 {'num': 'INTEGER','str': 'VARCHAR'} NULL NULL store_rejects=true
|
||||
|
||||
query IIIIIIII
|
||||
SELECT * EXCLUDE (scan_id, file_id) FROM reject_errors ORDER BY ALL;
|
||||
----
|
||||
3 15 15 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER'
|
||||
3 15 15 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER'
|
||||
5 23 23 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER'
|
||||
6 27 27 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER'
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
# Test with multiple rejects table in the same query, with different limits
|
||||
# (only one reject should be logged in right table)
|
||||
query IIII rowsort
|
||||
SELECT *
|
||||
FROM read_csv(
|
||||
'data/csv/error/mismatch/small1.csv',
|
||||
columns = {'num': 'INTEGER', 'str': 'VARCHAR'},
|
||||
store_rejects = true) as L
|
||||
JOIN read_csv(
|
||||
'data/csv/error/mismatch/small2.csv',
|
||||
columns = {'num': 'INTEGER', 'str': 'VARCHAR'},
|
||||
store_rejects = true, rejects_limit=1) as R
|
||||
ON L.num = R.num;
|
||||
----
|
||||
1 A 1 A
|
||||
3 C 3 C
|
||||
|
||||
|
||||
query IIIIIIII
|
||||
SELECT * EXCLUDE (scan_id, file_id) FROM reject_errors ORDER BY ALL;
|
||||
----
|
||||
3 15 15 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER'
|
||||
3 15 15 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER'
|
||||
6 27 27 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER'
|
||||
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIIII rowsort
|
||||
FROM read_csv('data/csv/rejects/dr_who.csv', columns={ 'date': 'DATE', 'datetime': 'TIMESTAMPTZ', 'time': 'TIME', 'timestamp': 'TIMESTAMP', 'time_tz': 'TIMETZ' }, store_rejects=true);
|
||||
----
|
||||
|
||||
|
||||
query IIIIIIII
|
||||
SELECT * EXCLUDE (scan_id, file_id) FROM reject_errors ORDER BY column_name;
|
||||
----
|
||||
2 38 38 1 date CAST not-a-date,not-a-datetime,not-a-time,not-a-timestamp,not-a-time-tz Error when converting column "date". Could not convert string "not-a-date" to 'DATE'
|
||||
2 38 49 2 datetime CAST not-a-date,not-a-datetime,not-a-time,not-a-timestamp,not-a-time-tz Error when converting column "datetime". Could not convert string "not-a-datetime" to 'TIMESTAMP WITH TIME ZONE'
|
||||
2 38 64 3 time CAST not-a-date,not-a-datetime,not-a-time,not-a-timestamp,not-a-time-tz Error when converting column "time". Could not convert string "not-a-time" to 'TIME'
|
||||
2 38 NULL 5 time_tz CAST not-a-date,not-a-datetime,not-a-time,not-a-timestamp,not-a-time-tz Error when converting column "time_tz". Could not convert string "not-a-time-tz" to 'TIME WITH TIME ZONE'
|
||||
2 38 75 4 timestamp CAST not-a-date,not-a-datetime,not-a-time,not-a-timestamp,not-a-time-tz Error when converting column "timestamp". Could not convert string "not-a-timestamp" to 'TIMESTAMP'
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
|
||||
# Test rejects table with comments
|
||||
statement ok
|
||||
FROM read_csv('data/csv/comments/error.csv', store_rejects = true, comment = '#');
|
||||
|
||||
query IIIIIIII
|
||||
SELECT * EXCLUDE (scan_id, file_id) FROM reject_errors ORDER BY column_name;
|
||||
----
|
||||
2726 10963 10963 1 a CAST oh no Error when converting column "a". Could not convert string "oh no" to 'BIGINT'
|
||||
2733 11009 11009 1 a CAST oh no Error when converting column "a". Could not convert string "oh no" to 'BIGINT'
|
||||
2726 10963 10968 1 b MISSING COLUMNS oh no Expected Number of Columns: 2 Found: 1
|
||||
2733 11009 11014 1 b MISSING COLUMNS oh no Expected Number of Columns: 2 Found: 1
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query II
|
||||
FROM read_csv('data/csv/error.csv', store_rejects=1, strict_mode=True);
|
||||
----
|
||||
true false
|
||||
|
||||
query IIIIIIII
|
||||
SELECT * EXCLUDE (scan_id, file_id) FROM reject_errors ORDER BY column_name;
|
||||
----
|
||||
4 36 39 1 column2 MISSING COLUMNS yes Expected Number of Columns: 2 Found: 1
|
||||
3 24 31 3 NULL TOO MANY COLUMNS yes,yes,yes Expected Number of Columns: 2 Found: 3
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
253
external/duckdb/test/sql/copy/csv/rejects/csv_rejects_two_tables.test
vendored
Normal file
253
external/duckdb/test/sql/copy/csv/rejects/csv_rejects_two_tables.test
vendored
Normal file
@@ -0,0 +1,253 @@
|
||||
# name: test/sql/copy/csv/rejects/csv_rejects_two_tables.test
|
||||
# group: [rejects]
|
||||
|
||||
require skip_reload
|
||||
|
||||
# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n
|
||||
require notwindows
|
||||
|
||||
# Ensure that we can get the schema if we reduce the sample size and ignore errors
|
||||
query IIIII
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
store_rejects=true);
|
||||
----
|
||||
BIGINT VARCHAR 11044 11044 2
|
||||
|
||||
|
||||
query IIIIIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_scans order by all;
|
||||
----
|
||||
0 data/csv/error/mismatch/big_bad.csv , (empty) (empty) \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL sample_size=1, store_rejects=true
|
||||
1 data/csv/error/mismatch/big_bad2.csv , (empty) (empty) \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL sample_size=1, store_rejects=true
|
||||
|
||||
query IIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors order by all;
|
||||
----
|
||||
0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT'
|
||||
0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT'
|
||||
1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT'
|
||||
1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT'
|
||||
|
||||
# Test giving the name of errors table
|
||||
statement error
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
rejects_table = 'rejects_errors_2');
|
||||
----
|
||||
Reject Scan Table name "reject_scans" is already in use. Either drop the used name(s), or give other name options in the CSV Reader function.
|
||||
|
||||
statement ok
|
||||
drop table reject_scans;
|
||||
|
||||
query IIIII
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
rejects_table = 'rejects_errors_2'
|
||||
);
|
||||
----
|
||||
BIGINT VARCHAR 11044 11044 2
|
||||
|
||||
query IIIIIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_scans order by all;
|
||||
----
|
||||
0 data/csv/error/mismatch/big_bad.csv , (empty) (empty) \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_table='rejects_errors_2', sample_size=1
|
||||
1 data/csv/error/mismatch/big_bad2.csv , (empty) (empty) \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_table='rejects_errors_2', sample_size=1
|
||||
|
||||
query IIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM rejects_errors_2 order by all;
|
||||
----
|
||||
0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT'
|
||||
0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT'
|
||||
1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT'
|
||||
1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT'
|
||||
|
||||
statement ok
|
||||
drop table reject_errors;
|
||||
|
||||
# Test giving the name of scans table
|
||||
query IIIII
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
rejects_scan = 'rejects_scan_2');
|
||||
----
|
||||
BIGINT VARCHAR 11044 11044 2
|
||||
|
||||
query IIIIIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM rejects_scan_2 order by all;
|
||||
----
|
||||
0 data/csv/error/mismatch/big_bad.csv , (empty) (empty) \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_2', sample_size=1
|
||||
1 data/csv/error/mismatch/big_bad2.csv , (empty) (empty) \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_2', sample_size=1
|
||||
|
||||
query IIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors order by all;
|
||||
----
|
||||
0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT'
|
||||
0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT'
|
||||
1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT'
|
||||
1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT'
|
||||
|
||||
# Test giving the name of both tables
|
||||
query IIIII
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
rejects_scan = 'rejects_scan_3',
|
||||
rejects_table = 'rejects_errors_3'
|
||||
);
|
||||
----
|
||||
BIGINT VARCHAR 11044 11044 2
|
||||
|
||||
query IIIIIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id)
|
||||
FROM rejects_scan_3 order by all;
|
||||
----
|
||||
0 data/csv/error/mismatch/big_bad.csv , (empty) (empty) \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_3', rejects_table='rejects_errors_3', sample_size=1
|
||||
1 data/csv/error/mismatch/big_bad2.csv , (empty) (empty) \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_3', rejects_table='rejects_errors_3', sample_size=1
|
||||
|
||||
query IIIIIIIII
|
||||
SELECT * EXCLUDE (scan_id) FROM rejects_errors_3 order by all;
|
||||
----
|
||||
0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT'
|
||||
0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT'
|
||||
1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT'
|
||||
1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT'
|
||||
|
||||
statement ok
|
||||
drop table reject_errors;
|
||||
|
||||
statement ok
|
||||
drop table reject_scans;
|
||||
|
||||
|
||||
# Test giving the name of an existing table to the errors table
|
||||
statement ok
|
||||
create temporary table t (a integer);
|
||||
|
||||
statement error
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
rejects_table = 't'
|
||||
);
|
||||
----
|
||||
Reject Error Table name "t" is already in use. Either drop the used name(s), or give other name options in the CSV Reader function.
|
||||
|
||||
# Test giving the name of an existing table to the scans table
|
||||
|
||||
statement error
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
rejects_scan = 't'
|
||||
);
|
||||
----
|
||||
Reject Scan Table name "t" is already in use. Either drop the used name(s), or give other name options in the CSV Reader function.
|
||||
|
||||
statement error
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
rejects_table = 't',
|
||||
rejects_scan = 't'
|
||||
);
|
||||
----
|
||||
The names of the rejects scan and rejects error tables can't be the same. Use different names for these tables.
|
||||
|
||||
# Test giving the name of the tables with store_rejects and/or ignore_errors set to false throws
|
||||
statement error
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
rejects_scan = 'rejects_scan_3',
|
||||
rejects_table = 'rejects_errors_3',
|
||||
ignore_errors = false
|
||||
);
|
||||
----
|
||||
STORE_REJECTS option is only supported when IGNORE_ERRORS is not manually set to false
|
||||
|
||||
statement error
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
store_rejects = true,
|
||||
ignore_errors = false
|
||||
);
|
||||
----
|
||||
STORE_REJECTS option is only supported when IGNORE_ERRORS is not manually set to false
|
||||
|
||||
statement error
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
rejects_table = 'rejects_errors_3',
|
||||
ignore_errors = false
|
||||
);
|
||||
----
|
||||
STORE_REJECTS option is only supported when IGNORE_ERRORS is not manually set to false
|
||||
|
||||
statement error
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
rejects_scan = 'rejects_scan_3',
|
||||
ignore_errors = false
|
||||
);
|
||||
----
|
||||
STORE_REJECTS option is only supported when IGNORE_ERRORS is not manually set to false
|
||||
|
||||
statement error
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
rejects_scan = 'rejects_scan_3',
|
||||
rejects_table = 'rejects_errors_3',
|
||||
store_rejects = false
|
||||
);
|
||||
----
|
||||
REJECTS_TABLE option is only supported when store_rejects is not manually set to false
|
||||
|
||||
statement error
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
rejects_table = 'rejects_errors_3',
|
||||
store_rejects = false
|
||||
);
|
||||
----
|
||||
REJECTS_TABLE option is only supported when store_rejects is not manually set to false
|
||||
|
||||
statement error
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
rejects_scan = 'rejects_scan_3',
|
||||
store_rejects = false
|
||||
);
|
||||
----
|
||||
REJECTS_SCAN option is only supported when store_rejects is not manually set to false
|
||||
|
||||
# Add a test where both tables have the same name (This should fail, because they both have the same name)
|
||||
statement error
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
rejects_scan = 'same_name_because_why_not',
|
||||
rejects_table = 'same_name_because_why_not',
|
||||
store_rejects = true
|
||||
);
|
||||
----
|
||||
The names of the rejects scan and rejects error tables can't be the same. Use different names for these tables.
|
||||
|
||||
# This hopefully doesn't fail because the names don't get registered if they fail.
|
||||
statement ok
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=1,
|
||||
rejects_scan = 'same_name_because_why_not',
|
||||
rejects_table = 'same_name_because_why_not_2',
|
||||
store_rejects = true);
|
||||
112
external/duckdb/test/sql/copy/csv/rejects/csv_unquoted_rejects.test
vendored
Normal file
112
external/duckdb/test/sql/copy/csv/rejects/csv_unquoted_rejects.test
vendored
Normal file
@@ -0,0 +1,112 @@
|
||||
# name: test/sql/copy/csv/rejects/csv_unquoted_rejects.test
|
||||
# description: Tests rejects tables on max line size parameter
|
||||
# group: [rejects]
|
||||
|
||||
require skip_reload
|
||||
|
||||
# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n
|
||||
require notwindows
|
||||
|
||||
query II
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/rejects/unquoted/basic.csv',
|
||||
columns = {'a': 'VARCHAR', 'b': 'INTEGER'},
|
||||
store_rejects=true, auto_detect=false, header = 1, quote = '"', escape = '"', strict_mode=true);
|
||||
----
|
||||
bla 1
|
||||
bla 2
|
||||
bla 3
|
||||
bla 1
|
||||
bla 2
|
||||
bla 3
|
||||
|
||||
query IIIIIIII rowsort
|
||||
SELECT regexp_replace(file_path, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line,line_byte_position, byte_position
|
||||
FROM reject_scans inner join reject_errors on (reject_scans.scan_id = reject_errors.scan_id and reject_scans.file_id = reject_errors.file_id);
|
||||
----
|
||||
data/csv/rejects/unquoted/basic.csv 5 1 a UNQUOTED VALUE "blaaaaaaaaaaaaaa"bla,4 29 29
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
query II
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/rejects/unquoted/unquoted_new_line.csv',
|
||||
columns = {'a': 'VARCHAR', 'b': 'INTEGER'},
|
||||
store_rejects=true, auto_detect=false, header = 1, quote = '"', escape = '"', strict_mode=true);
|
||||
----
|
||||
bla 1
|
||||
bla 2
|
||||
bla 3
|
||||
bla 1
|
||||
bla 2
|
||||
bla 3
|
||||
|
||||
query IIIIIII rowsort
|
||||
SELECT regexp_replace(file_path, '\\', '/', 'g'), line, column_idx, column_name, error_type, line_byte_position,byte_position
|
||||
FROM reject_scans inner join reject_errors on (reject_scans.scan_id = reject_errors.scan_id and reject_scans.file_id = reject_errors.file_id);
|
||||
----
|
||||
data/csv/rejects/unquoted/unquoted_new_line.csv 5 1 a UNQUOTED VALUE 29 29
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
query I
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/rejects/unquoted/unquoted_last_value.csv',
|
||||
columns = {'a': 'VARCHAR'},
|
||||
store_rejects=true, auto_detect=false, header = 0, quote = '"', escape = '"', strict_mode = true);
|
||||
----
|
||||
blaaaaaaaaaaaaaa
|
||||
bla
|
||||
bla
|
||||
bla
|
||||
|
||||
query IIIIIIII rowsort
|
||||
SELECT regexp_replace(file_path, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line,line_byte_position, byte_position
|
||||
FROM reject_scans inner join reject_errors on (reject_scans.scan_id = reject_errors.scan_id and reject_scans.file_id = reject_errors.file_id);
|
||||
----
|
||||
data/csv/rejects/unquoted/unquoted_last_value.csv 5 1 a UNQUOTED VALUE "bla 38 38
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
loop buffer_size 35 40
|
||||
|
||||
query II
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/rejects/unquoted/basic.csv',
|
||||
columns = {'a': 'VARCHAR', 'b': 'INTEGER'},
|
||||
buffer_size=${buffer_size},
|
||||
store_rejects=true, auto_detect=false, header = 1, quote = '"', escape = '"', strict_mode=true);
|
||||
----
|
||||
bla 1
|
||||
bla 2
|
||||
bla 3
|
||||
bla 1
|
||||
bla 2
|
||||
bla 3
|
||||
|
||||
|
||||
query IIIIIIII rowsort
|
||||
SELECT regexp_replace(file_path, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line,line_byte_position, byte_position
|
||||
FROM reject_scans inner join reject_errors on (reject_scans.scan_id = reject_errors.scan_id and reject_scans.file_id = reject_errors.file_id);
|
||||
----
|
||||
data/csv/rejects/unquoted/basic.csv 5 1 a UNQUOTED VALUE "blaaaaaaaaaaaaaa"bla,4 29 29
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
endloop
|
||||
113
external/duckdb/test/sql/copy/csv/rejects/test_invalid_parameters.test
vendored
Normal file
113
external/duckdb/test/sql/copy/csv/rejects/test_invalid_parameters.test
vendored
Normal file
@@ -0,0 +1,113 @@
|
||||
# name: test/sql/copy/csv/rejects/test_invalid_parameters.test
|
||||
# group: [rejects]
|
||||
|
||||
require skip_reload
|
||||
|
||||
# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n
|
||||
require notwindows
|
||||
|
||||
# Test invalid arguments
|
||||
statement error
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/error/mismatch/bad.csv',
|
||||
columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'},
|
||||
ignore_errors=false,
|
||||
store_rejects=true
|
||||
)
|
||||
----
|
||||
STORE_REJECTS option is only supported when IGNORE_ERRORS is not manually set to false
|
||||
|
||||
statement error
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/error/mismatch/bad.csv',
|
||||
columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'},
|
||||
ignore_errors=true,
|
||||
rejects_table='')
|
||||
----
|
||||
REJECTS_TABLE option cannot be empty
|
||||
|
||||
statement error
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/error/mismatch/bad.csv',
|
||||
columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'},
|
||||
ignore_errors=true,
|
||||
rejects_table='csv_rejects_table',
|
||||
union_by_name=true)
|
||||
----
|
||||
UNION_BY_NAME is set to true
|
||||
|
||||
statement error
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/error/mismatch/bad.csv',
|
||||
columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'},
|
||||
ignore_errors=true,
|
||||
rejects_limit=10)
|
||||
----
|
||||
REJECTS_LIMIT option is only supported when REJECTS_TABLE is set to a table name
|
||||
|
||||
statement error
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/error/mismatch/bad.csv',
|
||||
columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'},
|
||||
ignore_errors=true,
|
||||
rejects_table='csv_rejects_table',
|
||||
rejects_limit=-1)
|
||||
----
|
||||
REJECTS_LIMIT: cannot be negative
|
||||
|
||||
# Test invalid arguments
|
||||
statement error
|
||||
SELECT * FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/bad.csv',
|
||||
ignore_errors=false,
|
||||
rejects_table='csv_rejects_table'
|
||||
)
|
||||
----
|
||||
option is only supported when IGNORE_ERRORS is not manually set to false
|
||||
|
||||
statement error
|
||||
SELECT * FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/bad.csv',
|
||||
ignore_errors=true,
|
||||
rejects_table='')
|
||||
----
|
||||
REJECTS_TABLE option cannot be empty
|
||||
|
||||
statement error
|
||||
SELECT * FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/bad.csv',
|
||||
ignore_errors=true,
|
||||
rejects_table='csv_rejects_table',
|
||||
union_by_name=true)
|
||||
----
|
||||
UNION_BY_NAME is set to true
|
||||
|
||||
statement error
|
||||
SELECT * FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/bad.csv',
|
||||
ignore_errors=true,
|
||||
rejects_limit=10)
|
||||
----
|
||||
REJECTS_LIMIT option is only supported when REJECTS_TABLE is set to a table name
|
||||
|
||||
statement error
|
||||
SELECT * FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/bad.csv',
|
||||
ignore_errors=true,
|
||||
rejects_table='csv_rejects_table',
|
||||
rejects_limit=-1)
|
||||
----
|
||||
REJECTS_LIMIT: cannot be negative
|
||||
|
||||
|
||||
query III
|
||||
SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*) FROM read_csv_auto(
|
||||
'data/csv/error/mismatch/big_bad*.csv',
|
||||
sample_size=3000,
|
||||
rejects_table='csv_rejects_table',
|
||||
ignore_errors=true, header = 0);
|
||||
----
|
||||
VARCHAR VARCHAR 11048
|
||||
|
||||
statement ok
|
||||
DROP TABLE csv_rejects_table;
|
||||
20
external/duckdb/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test
vendored
Normal file
20
external/duckdb/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
# name: test/sql/copy/csv/rejects/test_invalid_utf_rejects.test
|
||||
# description: Test that invalid unicodes return correct info as rejects tables
|
||||
# group: [rejects]
|
||||
|
||||
require skip_reload
|
||||
|
||||
# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n
|
||||
require notwindows
|
||||
|
||||
statement ok
|
||||
from read_csv('data/csv/test/invalid_utf_big.csv',columns = {'col1': 'VARCHAR','col2': 'VARCHAR','col3': 'VARCHAR'},
|
||||
auto_detect=false, header = 0, delim = ',', store_rejects=true)
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL;
|
||||
----
|
||||
0 3001 54001 54007 2 col2 INVALID ENCODING valid,invalid_??_part,valid Invalid unicode (byte sequence mismatch) detected. This file is not utf-8 encoded.
|
||||
0 3012 54209 54221 3 col3 INVALID ENCODING valid,valid,invalid_??_part Invalid unicode (byte sequence mismatch) detected. This file is not utf-8 encoded.
|
||||
0 3023 54417 54423 2 col2 INVALID ENCODING valid,invalid_??_part,valid Invalid unicode (byte sequence mismatch) detected. This file is not utf-8 encoded.
|
||||
0 3034 54625 54637 3 col3 INVALID ENCODING valid,valid,invalid_??_part Invalid unicode (byte sequence mismatch) detected. This file is not utf-8 encoded.
|
||||
66
external/duckdb/test/sql/copy/csv/rejects/test_mixed.test
vendored
Normal file
66
external/duckdb/test/sql/copy/csv/rejects/test_mixed.test
vendored
Normal file
@@ -0,0 +1,66 @@
|
||||
# name: test/sql/copy/csv/rejects/test_mixed.test
|
||||
# description: Tests a mix of all possible CSV Errors
|
||||
# group: [rejects]
|
||||
|
||||
require skip_reload
|
||||
|
||||
# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n
|
||||
require notwindows
|
||||
|
||||
query III
|
||||
SELECT * FROM read_csv(
|
||||
'data/csv/rejects/frankstein/nightmare.csv',
|
||||
columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'VARCHAR'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=20, strict_mode=true);
|
||||
----
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
1 2 pedro
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL;
|
||||
----
|
||||
0 10 103 106 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 3 Found: 2
|
||||
0 14 143 154 4 NULL TOO MANY COLUMNS 1,2,"pedro",5 Expected Number of Columns: 3 Found: 4
|
||||
0 19 205 207 2 b CAST 1,bla,"pedro" Error when converting column "b". Could not convert string "bla" to 'INTEGER'
|
||||
0 22 243 247 3 c UNQUOTED VALUE 1,2,"pedro"bla Value with unterminated quote found.
|
||||
0 32 366 366 NULL NULL LINE SIZE OVER MAXIMUM 1,2,"pedro thiago timbo holanda" Maximum line size of 20 bytes exceeded. Actual Size:32 bytes.
|
||||
0 38 459 463 3 c INVALID ENCODING 1,2,"pedro??" Invalid unicode (byte sequence mismatch) detected. This file is not utf-8 encoded.
|
||||
383
external/duckdb/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test
vendored
Normal file
383
external/duckdb/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test
vendored
Normal file
@@ -0,0 +1,383 @@
|
||||
# name: test/sql/copy/csv/rejects/test_multiple_errors_same_line.test
|
||||
# description: Tests a mix of multiple errors and validate they get hit
|
||||
# group: [rejects]
|
||||
|
||||
require skip_reload
|
||||
|
||||
# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n
|
||||
require notwindows
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/cast_and_more_col.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, strict_mode=True);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
oogie boogie 3 2023-01-02 5
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL;
|
||||
----
|
||||
0 4 89 116 4 barks CAST oogie boogie,3, 2023-01-03, bla, 7 Error when converting column "barks". Could not convert string " bla" to 'INTEGER'
|
||||
0 4 89 120 5 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7 Expected Number of Columns: 4 Found: 5
|
||||
0 5 124 151 4 barks CAST oogie boogie,3, 2023-01-03, bla, 7, 8 Error when converting column "barks". Could not convert string " bla" to 'INTEGER'
|
||||
0 5 124 155 5 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7, 8 Expected Number of Columns: 4 Found: 5
|
||||
0 5 124 158 6 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7, 8 Expected Number of Columns: 4 Found: 6
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/multiple_cast_implicit.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, strict_mode=True);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
oogie boogie 3 2023-01-02 5
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL;
|
||||
----
|
||||
0 4 89 102 2 age CAST oogie boogie,bla_2, 2023-01-02, bla_1 Error when converting column "age". Could not convert string "bla_2" to 'INTEGER'
|
||||
0 4 89 120 4 barks CAST oogie boogie,bla_2, 2023-01-02, bla_1 Error when converting column "barks". Could not convert string " bla_1" to 'INTEGER'
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/multiple_casts_flush.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'tomorrow': 'DATE'},
|
||||
store_rejects = true, auto_detect=false, header = 1);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2023-01-02
|
||||
oogie boogie 3 2023-01-02 2023-01-03
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL;
|
||||
----
|
||||
0 4 110 125 3 current_day CAST oogie boogie,3, bla_2, bla_1 Error when converting column "current_day". Could not convert string " bla_2" to 'DATE'
|
||||
0 4 110 132 4 tomorrow CAST oogie boogie,3, bla_2, bla_1 Error when converting column "tomorrow". Could not convert string " bla_1" to 'DATE'
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/multiple_casts_mixed.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
oogie boogie 3 2023-01-02 5
|
||||
|
||||
# FIXME: This will not present the both cast errors :'(, should be alleviated the more types we add to implicit casting
|
||||
query IIIIIIIII rowsort
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL;
|
||||
----
|
||||
0 4 89 104 3 current_day CAST oogie boogie,3, bla_2, bla_1 Error when converting column "current_day". Could not convert string " bla_2" to 'DATE'
|
||||
0 4 89 111 4 barks CAST oogie boogie,3, bla_2, bla_1 Error when converting column "barks". Could not convert string " bla_1" to 'INTEGER'
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/cast_and_less_col.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
oogie boogie 3 2023-01-02 5
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL;
|
||||
----
|
||||
0 4 89 102 2 age CAST oogie boogie,bla, 2023-01-03 Error when converting column "age". Could not convert string "bla" to 'INTEGER'
|
||||
0 4 89 117 3 barks MISSING COLUMNS oogie boogie,bla, 2023-01-03 Expected Number of Columns: 4 Found: 3
|
||||
0 5 118 131 2 age CAST oogie boogie,bla Error when converting column "age". Could not convert string "bla" to 'INTEGER'
|
||||
0 5 118 134 2 current_day MISSING COLUMNS oogie boogie,bla Expected Number of Columns: 4 Found: 2
|
||||
0 5 118 134 3 barks MISSING COLUMNS oogie boogie,bla Expected Number of Columns: 4 Found: 3
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/cast_and_maxline.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=40);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
oogie boogie 3 2023-01-02 5
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL;
|
||||
----
|
||||
0 4 89 138 2 age CAST oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03, 4 Error when converting column "age". Could not convert string "bla" to 'INTEGER'
|
||||
0 4 89 89 NULL NULL LINE SIZE OVER MAXIMUM oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03, 4 Maximum line size of 40 bytes exceeded. Actual Size:67 bytes.
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/less_col_and_max_line.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=40);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
oogie boogie 3 2023-01-02 5
|
||||
|
||||
query IIIIIIIII
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position;
|
||||
----
|
||||
0 4 89 89 NULL NULL LINE SIZE OVER MAXIMUM oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03 Maximum line size of 40 bytes exceeded. Actual Size:64 bytes.
|
||||
0 4 89 138 2 age CAST oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03 Error when converting column "age". Could not convert string "bla" to 'INTEGER'
|
||||
0 4 89 153 3 barks MISSING COLUMNS oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03 Expected Number of Columns: 4 Found: 3
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/more_col_and_max_line.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=40, strict_mode=True);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
oogie boogie 3 2023-01-02 5
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position;
|
||||
----
|
||||
0 4 89 138 2 age CAST oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03,4, bla Error when converting column "age". Could not convert string "bla" to 'INTEGER'
|
||||
0 4 89 155 5 NULL TOO MANY COLUMNS oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03,4, bla Expected Number of Columns: 4 Found: 5
|
||||
0 4 89 89 NULL NULL LINE SIZE OVER MAXIMUM oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03,4, bla Maximum line size of 40 bytes exceeded. Actual Size:71 bytes.
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/unquoted_cast.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=40, strict_mode=true);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
oogie boogie 3 2023-01-02 7
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position;
|
||||
----
|
||||
0 3 59 59 1 name UNQUOTED VALUE "oogie boogie"bla,bla, 2023-01-02, 5 Value with unterminated quote found.
|
||||
0 3 59 77 2 age CAST "oogie boogie"bla,bla, 2023-01-02, 5 Error when converting column "age". Could not convert string "bla" to 'INTEGER'
|
||||
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/unquoted_less.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=40, strict_mode=true);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
oogie boogie 3 2023-01-02 7
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position;
|
||||
----
|
||||
0 3 59 59 1 name UNQUOTED VALUE "oogie boogie"bla,4, 2023-01-02 Value with unterminated quote found.
|
||||
0 3 59 90 3 barks MISSING COLUMNS "oogie boogie"bla,4, 2023-01-02 Expected Number of Columns: 4 Found: 3
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/unquoted_maxline.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=40, strict_mode=true);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
oogie boogie 3 2023-01-02 7
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position;
|
||||
----
|
||||
0 3 59 59 1 name UNQUOTED VALUE "oogie boogieoogie boogieoogie boogieoogie boogie"bla,4, 2023-01-02, 5 Value with unterminated quote found.
|
||||
0 3 59 59 NULL NULL LINE SIZE OVER MAXIMUM "oogie boogieoogie boogieoogie boogieoogie boogie"bla,4, 2023-01-02, 5 Maximum line size of 40 bytes exceeded. Actual Size:70 bytes.
|
||||
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/unquoted_more.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=40, strict_mode=true);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
oogie boogie 3 2023-01-02 7
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position;
|
||||
----
|
||||
0 3 59 59 1 name UNQUOTED VALUE "oogie boogie"bla,4, 2023-01-02, 5, 8 Value with unterminated quote found.
|
||||
0 3 59 93 5 NULL TOO MANY COLUMNS "oogie boogie"bla,4, 2023-01-02, 5, 8 Expected Number of Columns: 4 Found: 5
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/invalid_utf_cast.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=40);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position;
|
||||
----
|
||||
0 3 59 59 1 name INVALID ENCODING oogie bo??gie,bla, 2023-01-01, 2 Invalid unicode (byte sequence mismatch) detected. This file is not utf-8 encoded.
|
||||
0 3 59 73 2 age CAST oogie bo??gie,bla, 2023-01-01, 2 Error when converting column "age". Could not convert string "bla" to 'INTEGER'
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/invalid_utf_less.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=40);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position;
|
||||
----
|
||||
0 3 59 59 1 name INVALID ENCODING oogie bo??gie,3, 2023-01-01 Invalid unicode (byte sequence mismatch) detected. This file is not utf-8 encoded.
|
||||
0 3 59 86 3 barks MISSING COLUMNS oogie bo??gie,3, 2023-01-01 Expected Number of Columns: 4 Found: 3
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/invalid_utf_max_line.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=40, strict_mode=True);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position, error_message;
|
||||
----
|
||||
0 3 59 125 5 NULL TOO MANY COLUMNS oogie bo??gieoogie boogieoogie boogieoogie boogie,3, 2023-01-01, 2, 5 Expected Number of Columns: 4 Found: 5
|
||||
0 3 59 59 1 name INVALID ENCODING oogie bo??gieoogie boogieoogie boogieoogie boogie,3, 2023-01-01, 2, 5 Invalid unicode (byte sequence mismatch) detected. This file is not utf-8 encoded.
|
||||
0 3 59 59 NULL NULL LINE SIZE OVER MAXIMUM oogie bo??gieoogie boogieoogie boogieoogie boogie,3, 2023-01-01, 2, 5 Maximum line size of 40 bytes exceeded. Actual Size:69 bytes.
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/invalid_utf_more.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=40, strict_mode=True);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position;
|
||||
----
|
||||
0 3 59 59 1 name INVALID ENCODING oogie bo??gie,3, 2023-01-01, 2, 5 Invalid unicode (byte sequence mismatch) detected. This file is not utf-8 encoded.
|
||||
0 3 59 89 5 NULL TOO MANY COLUMNS oogie bo??gie,3, 2023-01-01, 2, 5 Expected Number of Columns: 4 Found: 5
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/invalid_utf_unquoted.csv',
|
||||
columns = {'name': 'VARCHAR', 'last_name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=40, strict_mode=true);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position;
|
||||
----
|
||||
0 3 71 71 1 name UNQUOTED VALUE "oogie"bla, bo??gie,3, 2023-01-01, 2 Value with unterminated quote found.
|
||||
0 3 71 82 2 last_name INVALID ENCODING "oogie"bla, bo??gie,3, 2023-01-01, 2 Invalid unicode (byte sequence mismatch) detected. This file is not utf-8 encoded.
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_errors;
|
||||
|
||||
statement ok
|
||||
DROP TABLE reject_scans;
|
||||
|
||||
query IIII
|
||||
FROM read_csv('data/csv/rejects/multiple_errors/multiple_errors.csv',
|
||||
columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'},
|
||||
store_rejects = true, auto_detect=false, header = 1, max_line_size=40, strict_mode=true);
|
||||
----
|
||||
oogie boogie 3 2023-01-01 2
|
||||
oogie boogie 3 2023-01-02 5
|
||||
|
||||
query IIIIIIIII rowsort
|
||||
SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL;
|
||||
----
|
||||
0 4 89 116 4 barks CAST oogie boogie,3, 2023-01-03, bla, 7 Error when converting column "barks". Could not convert string " bla" to 'INTEGER'
|
||||
0 4 89 120 5 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7 Expected Number of Columns: 4 Found: 5
|
||||
0 5 124 139 3 current_day CAST oogie boogie,3, bla, bla, 7 Error when converting column "current_day". Could not convert string " bla" to 'DATE'
|
||||
0 5 124 144 4 barks CAST oogie boogie,3, bla, bla, 7 Error when converting column "barks". Could not convert string " bla" to 'INTEGER'
|
||||
0 5 124 148 5 NULL TOO MANY COLUMNS oogie boogie,3, bla, bla, 7 Expected Number of Columns: 4 Found: 5
|
||||
0 6 152 152 1 name UNQUOTED VALUE "oogie boogie"bla,3, 2023-01-04 Value with unterminated quote found.
|
||||
0 6 152 183 3 barks MISSING COLUMNS "oogie boogie"bla,3, 2023-01-04 Expected Number of Columns: 4 Found: 3
|
||||
0 7 184 199 3 current_day CAST oogie boogie,3, bla Error when converting column "current_day". Could not convert string " bla" to 'DATE'
|
||||
0 7 184 203 3 barks MISSING COLUMNS oogie boogie,3, bla Expected Number of Columns: 4 Found: 3
|
||||
0 8 204 204 NULL NULL LINE SIZE OVER MAXIMUM oogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogie,3, bla Maximum line size of 40 bytes exceeded. Actual Size:91 bytes.
|
||||
0 8 204 291 3 current_day CAST oogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogie,3, bla Error when converting column "current_day". Could not convert string " bla" to 'DATE'
|
||||
0 8 204 295 3 barks MISSING COLUMNS oogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogie,3, bla Expected Number of Columns: 4 Found: 3
|
||||
Reference in New Issue
Block a user