should be it
This commit is contained in:
85
external/duckdb/test/sql/storage/compression/string/big_strings.test
vendored
Normal file
85
external/duckdb/test/sql/storage/compression/string/big_strings.test
vendored
Normal file
@@ -0,0 +1,85 @@
|
||||
# name: test/sql/storage/compression/string/big_strings.test
|
||||
# description: Test string compression with big strings, which are currently unsupported by the compression algorithms
|
||||
# group: [string]
|
||||
|
||||
statement ok
|
||||
ATTACH '__TEST_DIR__/test_big_strings_new.db' AS db_v13 (STORAGE_VERSION 'v1.3.0');
|
||||
|
||||
statement ok
|
||||
ATTACH '__TEST_DIR__/test_big_strings_old.db' AS db_v1 (STORAGE_VERSION 'v1.0.0');
|
||||
|
||||
foreach compression fsst dictionary dict_fsst
|
||||
|
||||
statement ok
|
||||
USE db_v1
|
||||
|
||||
onlyif compression=dict_fsst
|
||||
statement ok
|
||||
USE db_v13
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='${compression}'
|
||||
|
||||
# Baseline, non big string, dictionary compression should work
|
||||
statement ok
|
||||
CREATE TABLE normal_string (a VARCHAR);
|
||||
|
||||
statement ok
|
||||
INSERT INTO normal_string values (repeat('a', 3900));
|
||||
INSERT INTO normal_string values (repeat('b', 3900));
|
||||
INSERT INTO normal_string values (repeat('a', 3900));
|
||||
INSERT INTO normal_string values (repeat('b', 3900));
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query III
|
||||
SELECT list_aggr(str_split(a,''),'min'), list_aggr(str_split(a,''),'min'), strlen(a) from normal_string
|
||||
----
|
||||
a a 3900
|
||||
b b 3900
|
||||
a a 3900
|
||||
b b 3900
|
||||
|
||||
query I
|
||||
SELECT lower(compression)='${compression}' FROM pragma_storage_info('normal_string') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
|
||||
----
|
||||
1
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='uncompressed'
|
||||
|
||||
# Big string, should disable string compression
|
||||
statement ok
|
||||
CREATE TABLE big_string (a VARCHAR);
|
||||
|
||||
statement ok
|
||||
INSERT INTO big_string values (repeat('a', 8000));
|
||||
INSERT INTO big_string values (repeat('b', 8000));
|
||||
INSERT INTO big_string values (repeat('a', 8000));
|
||||
INSERT INTO big_string values (repeat('b', 8000));
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query III
|
||||
SELECT list_aggr(str_split(a,''),'min'), list_aggr(str_split(a,''),'min'), strlen(a) from big_string
|
||||
----
|
||||
a a 8000
|
||||
b b 8000
|
||||
a a 8000
|
||||
b b 8000
|
||||
|
||||
query I
|
||||
SELECT lower(compression) FROM pragma_storage_info('big_string') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
|
||||
----
|
||||
uncompressed
|
||||
|
||||
statement ok
|
||||
DROP TABLE big_string;
|
||||
|
||||
statement ok
|
||||
DROP TABLE normal_string;
|
||||
|
||||
endloop
|
||||
|
||||
131
external/duckdb/test/sql/storage/compression/string/blob.test
vendored
Normal file
131
external/duckdb/test/sql/storage/compression/string/blob.test
vendored
Normal file
@@ -0,0 +1,131 @@
|
||||
# name: test/sql/storage/compression/string/blob.test
|
||||
# description: blob tests
|
||||
# group: [string]
|
||||
|
||||
statement ok
|
||||
ATTACH '__TEST_DIR__/test_blob_new.db' AS db_v13 (STORAGE_VERSION 'v1.3.0');
|
||||
|
||||
statement ok
|
||||
ATTACH '__TEST_DIR__/test_blob_old.db' AS db_v1 (STORAGE_VERSION 'v1.0.0');
|
||||
|
||||
foreach compression fsst dictionary dict_fsst
|
||||
|
||||
statement ok
|
||||
USE db_v1
|
||||
|
||||
onlyif compression=dict_fsst
|
||||
statement ok
|
||||
USE db_v13
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='${compression}'
|
||||
|
||||
statement ok
|
||||
PRAGMA enable_verification
|
||||
|
||||
statement ok
|
||||
CREATE TABLE blobs (b BYTEA);
|
||||
|
||||
# Insert valid hex strings
|
||||
statement ok
|
||||
INSERT INTO blobs VALUES('\xaa\xff\xaa'), ('\xAA\xFF\xAA\xAA\xFF\xAA'), ('\xAA\xFF\xAA\xAA\xFF\xAA\xAA\xFF\xAA')
|
||||
|
||||
query T
|
||||
SELECT * FROM blobs
|
||||
----
|
||||
\xAA\xFF\xAA
|
||||
\xAA\xFF\xAA\xAA\xFF\xAA
|
||||
\xAA\xFF\xAA\xAA\xFF\xAA\xAA\xFF\xAA
|
||||
|
||||
# Insert valid hex strings, lower case
|
||||
statement ok
|
||||
DELETE FROM blobs
|
||||
|
||||
statement ok
|
||||
INSERT INTO blobs VALUES('\xaa\xff\xaa'), ('\xaa\xff\xaa\xaa\xff\xaa'), ('\xaa\xff\xaa\xaa\xff\xaa\xaa\xff\xaa')
|
||||
|
||||
query T
|
||||
SELECT * FROM blobs
|
||||
----
|
||||
\xAA\xFF\xAA
|
||||
\xAA\xFF\xAA\xAA\xFF\xAA
|
||||
\xAA\xFF\xAA\xAA\xFF\xAA\xAA\xFF\xAA
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
SELECT lower(compression)='${compression}' FROM pragma_storage_info('blobs') WHERE segment_type ILIKE 'BLOB' LIMIT 1
|
||||
----
|
||||
1
|
||||
|
||||
# Insert valid hex strings with number and letters
|
||||
statement ok
|
||||
DELETE FROM blobs
|
||||
|
||||
statement ok
|
||||
INSERT INTO blobs VALUES('\xaa1199'), ('\xaa1199aa1199'), ('\xaa1199aa1199aa1199')
|
||||
|
||||
query T
|
||||
SELECT * FROM blobs
|
||||
----
|
||||
\xAA1199
|
||||
\xAA1199aa1199
|
||||
\xAA1199aa1199aa1199
|
||||
|
||||
# Insert invalid hex strings (invalid hex chars: G, H, I)
|
||||
statement error
|
||||
INSERT INTO blobs VALUES('\xGA\xFF\xAA')
|
||||
----
|
||||
|
||||
# Insert invalid hex strings (odd # of chars)
|
||||
statement error
|
||||
INSERT INTO blobs VALUES('\xA')
|
||||
----
|
||||
|
||||
statement error
|
||||
INSERT INTO blobs VALUES('\xAA\xA')
|
||||
----
|
||||
|
||||
statement error
|
||||
INSERT INTO blobs VALUES('blablabla\x')
|
||||
----
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
SELECT lower(compression)='${compression}' FROM pragma_storage_info('blobs') WHERE segment_type ILIKE 'BLOB' LIMIT 1
|
||||
----
|
||||
1
|
||||
|
||||
statement ok
|
||||
CREATE TABLE blob_empty (b BYTEA);
|
||||
|
||||
statement ok
|
||||
INSERT INTO blob_empty VALUES(''), (''::BLOB)
|
||||
|
||||
statement ok
|
||||
INSERT INTO blob_empty VALUES(NULL), (NULL::BLOB)
|
||||
|
||||
query T
|
||||
SELECT * FROM blob_empty
|
||||
----
|
||||
(empty)
|
||||
(empty)
|
||||
NULL
|
||||
NULL
|
||||
|
||||
# FSST will get skipped for only empty/null strings
|
||||
query I
|
||||
SELECT lower(compression)!='fsst' FROM pragma_storage_info('blob_empty') WHERE segment_type ILIKE 'BLOB' LIMIT 1
|
||||
----
|
||||
1
|
||||
|
||||
statement ok
|
||||
DROP TABLE blobs;
|
||||
|
||||
statement ok
|
||||
DROP TABLE blob_empty;
|
||||
|
||||
endloop
|
||||
78
external/duckdb/test/sql/storage/compression/string/empty.test
vendored
Normal file
78
external/duckdb/test/sql/storage/compression/string/empty.test
vendored
Normal file
@@ -0,0 +1,78 @@
|
||||
# name: test/sql/storage/compression/string/empty.test
|
||||
# description: Test empty strings with string compression
|
||||
# group: [string]
|
||||
|
||||
require no_latest_storage
|
||||
|
||||
require vector_size 512
|
||||
|
||||
statement ok
|
||||
ATTACH '__TEST_DIR__/test_empty_new.db' AS db_v13 (STORAGE_VERSION 'v1.3.0');
|
||||
|
||||
statement ok
|
||||
ATTACH '__TEST_DIR__/test_empty_old.db' AS db_v1 (STORAGE_VERSION 'v1.0.0');
|
||||
|
||||
foreach compression fsst dictionary dict_fsst
|
||||
|
||||
statement ok
|
||||
USE db_v1
|
||||
|
||||
onlyif compression=dict_fsst
|
||||
statement ok
|
||||
USE db_v13
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='${compression}'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_empty (a VARCHAR);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test_empty VALUES('');
|
||||
INSERT INTO test_empty VALUES('A');
|
||||
INSERT INTO test_empty VALUES(NULL);
|
||||
INSERT INTO test_empty VALUES('B');
|
||||
INSERT INTO test_empty VALUES('');
|
||||
INSERT INTO test_empty VALUES('A');
|
||||
|
||||
statement ok
|
||||
CHECKPOINT
|
||||
|
||||
query I
|
||||
select * from test_empty;
|
||||
----
|
||||
(empty)
|
||||
A
|
||||
NULL
|
||||
B
|
||||
(empty)
|
||||
A
|
||||
|
||||
query I
|
||||
SELECT lower(compression)='${compression}' FROM pragma_storage_info('test_empty') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
|
||||
----
|
||||
1
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_empty_large AS SELECT '' as a from range(0,10000) union all select 'A' union all select '';
|
||||
|
||||
statement ok
|
||||
CHECKPOINT
|
||||
|
||||
query III
|
||||
select count(*), min(a[1]), max(a[1]) from test_empty_large limit 5;
|
||||
----
|
||||
10002 (empty) A
|
||||
|
||||
query I
|
||||
SELECT lower(compression)='${compression}' FROM pragma_storage_info('test_empty_large') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
|
||||
----
|
||||
1
|
||||
|
||||
statement ok
|
||||
DROP TABLE test_empty
|
||||
|
||||
statement ok
|
||||
DROP TABLE test_empty_large
|
||||
|
||||
endloop
|
||||
53
external/duckdb/test/sql/storage/compression/string/filter_pushdown.test
vendored
Normal file
53
external/duckdb/test/sql/storage/compression/string/filter_pushdown.test
vendored
Normal file
@@ -0,0 +1,53 @@
|
||||
# name: test/sql/storage/compression/string/filter_pushdown.test
|
||||
# description: Filter pushdown with string compressed columns
|
||||
# group: [string]
|
||||
|
||||
statement ok
|
||||
ATTACH '__TEST_DIR__/test_filter_pushdown_new.db' AS db_v13 (STORAGE_VERSION 'v1.3.0');
|
||||
|
||||
statement ok
|
||||
ATTACH '__TEST_DIR__/test_filter_pushdown_old.db' AS db_v1 (STORAGE_VERSION 'v1.0.0');
|
||||
|
||||
foreach compression fsst dictionary dict_fsst
|
||||
|
||||
statement ok
|
||||
USE db_v1
|
||||
|
||||
onlyif compression=dict_fsst
|
||||
statement ok
|
||||
USE db_v13
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='${compression}'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test (id INT, col VARCHAR)
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::INT id, concat('BLEEPBLOOP-', (i%10)::VARCHAR) col FROM range(10000) tbl(i)
|
||||
|
||||
statement ok
|
||||
CHECKPOINT
|
||||
|
||||
# filter on the dictionary column
|
||||
query III
|
||||
SELECT MIN(col), MAX(col), COUNT(*) FROM test WHERE col >= 'BLEEPBLOOP-5'
|
||||
----
|
||||
BLEEPBLOOP-5 BLEEPBLOOP-9 5000
|
||||
|
||||
# filter on non-dictionary column
|
||||
query IIIII
|
||||
SELECT MIN(id), MAX(id), MIN(col), MAX(col), COUNT(*) FROM test WHERE id='5000'
|
||||
----
|
||||
5000 5000 BLEEPBLOOP-0 BLEEPBLOOP-0 1
|
||||
|
||||
query I
|
||||
SELECT lower(compression)='${compression}' FROM pragma_storage_info('test') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
|
||||
----
|
||||
1
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
endloop
|
||||
|
||||
65
external/duckdb/test/sql/storage/compression/string/index_fetch.test
vendored
Normal file
65
external/duckdb/test/sql/storage/compression/string/index_fetch.test
vendored
Normal file
@@ -0,0 +1,65 @@
|
||||
# name: test/sql/storage/compression/string/index_fetch.test
|
||||
# description: Fetch from compressed string column with index
|
||||
# group: [string]
|
||||
|
||||
statement ok
|
||||
ATTACH '__TEST_DIR__/test_index_fetch_new.db' AS db_v13 (STORAGE_VERSION 'v1.3.0');
|
||||
|
||||
statement ok
|
||||
ATTACH '__TEST_DIR__/test_index_fetch_old.db' AS db_v1 (STORAGE_VERSION 'v1.0.0');
|
||||
|
||||
foreach compression fsst dictionary dict_fsst
|
||||
|
||||
statement ok
|
||||
USE db_v1
|
||||
|
||||
onlyif compression=dict_fsst
|
||||
statement ok
|
||||
USE db_v13
|
||||
|
||||
statement ok
|
||||
drop type if exists test_result
|
||||
|
||||
statement ok
|
||||
create type test_result as UNION(
|
||||
ok BOOL,
|
||||
err STRUCT(
|
||||
expected VARCHAR,
|
||||
actual VARCHAR
|
||||
)
|
||||
);
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='${compression}'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test(id INTEGER PRIMARY KEY, col VARCHAR)
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i id, i::VARCHAR b FROM range(10000) tbl(i)
|
||||
|
||||
statement ok
|
||||
CHECKPOINT
|
||||
|
||||
query IIIIII
|
||||
SELECT MIN(id), MAX(id), SUM(col::INT), MIN(col::INT), MAX(col::INT), COUNT(*) FROM test WHERE id=5000
|
||||
----
|
||||
5000 5000 5000 5000 5000 1
|
||||
|
||||
query I
|
||||
SELECT
|
||||
CASE WHEN lower(compression)='${compression}'
|
||||
THEN true::test_result
|
||||
ELSE {
|
||||
'expected': '${compression}',
|
||||
'actual': lower(compression)
|
||||
}::test_result
|
||||
END
|
||||
FROM pragma_storage_info('test') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
|
||||
----
|
||||
true
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
endloop
|
||||
56
external/duckdb/test/sql/storage/compression/string/large.test_slow
vendored
Normal file
56
external/duckdb/test/sql/storage/compression/string/large.test_slow
vendored
Normal file
@@ -0,0 +1,56 @@
|
||||
# name: test/sql/storage/compression/string/large.test_slow
|
||||
# description: Test large storage with string compression (many row groups)
|
||||
# group: [string]
|
||||
|
||||
statement ok
|
||||
PRAGMA enable_verification
|
||||
|
||||
foreach compression fsst dictionary dict_fsst
|
||||
|
||||
statement ok
|
||||
SET storage_compatibility_version='v1.0.0'
|
||||
|
||||
onlyif compression=dict_fsst
|
||||
statement ok
|
||||
SET storage_compatibility_version='v1.3.0'
|
||||
|
||||
load __TEST_DIR__/test_large_${compression}.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='${compression}'
|
||||
|
||||
# simple dictionary compressed column with few values
|
||||
statement ok
|
||||
CREATE TABLE test (a VARCHAR);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT (i::VARCHAR) FROM range(0, 1000000) tbl(i);
|
||||
INSERT INTO test SELECT NULL FROM range(0, 1000000) tbl(i);
|
||||
INSERT INTO test SELECT (i::VARCHAR) FROM range(0, 1000000) tbl(i);
|
||||
INSERT INTO test SELECT ''::VARCHAR FROM range(0, 1000000) tbl(i);
|
||||
INSERT INTO test SELECT (i::VARCHAR) FROM range(0, 1000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query IIII
|
||||
SELECT SUM(a::INT), MIN(a::INT), MAX(a::INT), COUNT(*) FROM test where a NOT NULL and a != ''
|
||||
----
|
||||
1499998500000 0 999999 3000000
|
||||
|
||||
restart
|
||||
|
||||
query IIII
|
||||
SELECT SUM(a::INT), MIN(a::INT), MAX(a::INT), COUNT(*) FROM test where a NOT NULL and a != ''
|
||||
----
|
||||
1499998500000 0 999999 3000000
|
||||
|
||||
query I
|
||||
SELECT lower(compression)='${compression}' FROM pragma_storage_info('test') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
|
||||
----
|
||||
1
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
endloop
|
||||
84
external/duckdb/test/sql/storage/compression/string/lists.test_slow
vendored
Normal file
84
external/duckdb/test/sql/storage/compression/string/lists.test_slow
vendored
Normal file
@@ -0,0 +1,84 @@
|
||||
# name: test/sql/storage/compression/string/lists.test_slow
|
||||
# description: Test storage with string compression inside lists
|
||||
# group: [string]
|
||||
|
||||
statement ok
|
||||
ATTACH '__TEST_DIR__/test_large_new.db' AS db_v13 (STORAGE_VERSION 'v1.3.0');
|
||||
|
||||
statement ok
|
||||
ATTACH '__TEST_DIR__/test_large_old.db' AS db_v1 (STORAGE_VERSION 'v1.0.0');
|
||||
|
||||
foreach compression fsst dictionary dict_fsst
|
||||
|
||||
statement ok
|
||||
USE db_v1
|
||||
|
||||
onlyif compression=dict_fsst
|
||||
statement ok
|
||||
USE db_v13
|
||||
|
||||
statement ok
|
||||
pragma threads=1
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='${compression}'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test (id INTEGER, l VARCHAR[]);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i, case when (i//1000)%2=0 then [1::VARCHAR, 1::VARCHAR, 1::VARCHAR] else [2::VARCHAR, 2::VARCHAR] end FROM range(200000) tbl(i)
|
||||
|
||||
# full unnest
|
||||
query II
|
||||
SELECT COUNT(*), SUM(i::INT) FROM (SELECT UNNEST(l) FROM test) tbl(i)
|
||||
----
|
||||
500000 700000
|
||||
|
||||
# filters/skips
|
||||
query II
|
||||
SELECT COUNT(*), SUM(i::INT) FROM (SELECT UNNEST(l) FROM test WHERE id>=5000 AND id<6000) tbl(i)
|
||||
----
|
||||
2000 4000
|
||||
|
||||
# zonemaps
|
||||
query II
|
||||
SELECT COUNT(*), SUM(i::INT) FROM (SELECT UNNEST(l) FROM test WHERE id>=150000 AND id<160000) tbl(i)
|
||||
----
|
||||
25000 35000
|
||||
|
||||
statement ok
|
||||
CREATE INDEX i_index ON test(id)
|
||||
|
||||
# index lookup in lists
|
||||
query II
|
||||
SELECT * FROM test WHERE id=150001
|
||||
----
|
||||
150001 [1, 1, 1]
|
||||
|
||||
# large lists
|
||||
statement ok
|
||||
CREATE TABLE test_large_list AS SELECT i%100 AS id, LIST((-i)::VARCHAR) AS list FROM range(0,100000) tbl(i) GROUP BY id;
|
||||
|
||||
query II
|
||||
SELECT COUNT(*), SUM(i::INT) FROM (SELECT UNNEST(list) FROM test_large_list) tbl(i)
|
||||
----
|
||||
100000 -4999950000
|
||||
|
||||
query I
|
||||
SELECT lower(compression)='${compression}' FROM pragma_storage_info('test_large_list') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
|
||||
----
|
||||
1
|
||||
|
||||
query I
|
||||
SELECT lower(compression)='${compression}' FROM pragma_storage_info('test') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
|
||||
----
|
||||
1
|
||||
|
||||
statement ok
|
||||
DROP TABLE test_large_list
|
||||
|
||||
statement ok
|
||||
DROP TABLE test
|
||||
|
||||
endloop
|
||||
29
external/duckdb/test/sql/storage/compression/string/lists_vector_type_assertion.test_slow
vendored
Normal file
29
external/duckdb/test/sql/storage/compression/string/lists_vector_type_assertion.test_slow
vendored
Normal file
@@ -0,0 +1,29 @@
|
||||
# name: test/sql/storage/compression/string/lists_vector_type_assertion.test_slow
|
||||
# description: For smaller block sizes, this test ensures that the vector type of result vectors does not change for partial scans
|
||||
# group: [string]
|
||||
|
||||
load __TEST_DIR__/test_string_compression.db
|
||||
|
||||
statement ok
|
||||
SET enable_fsst_vectors='true'
|
||||
|
||||
statement ok
|
||||
pragma threads=1
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='fsst'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test (l VARCHAR[]);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT
|
||||
CASE WHEN (i//1000)%2=0
|
||||
THEN [1::VARCHAR, 1::VARCHAR, 1::VARCHAR]
|
||||
ELSE [2::VARCHAR, 2::VARCHAR] END
|
||||
FROM range(200000) tbl(i)
|
||||
|
||||
query II
|
||||
SELECT COUNT(*), SUM(i::INT) FROM (SELECT UNNEST(l) FROM test) tbl(i)
|
||||
----
|
||||
500000 700000
|
||||
49
external/duckdb/test/sql/storage/compression/string/medium.test
vendored
Normal file
49
external/duckdb/test/sql/storage/compression/string/medium.test
vendored
Normal file
@@ -0,0 +1,49 @@
|
||||
# name: test/sql/storage/compression/string/medium.test
|
||||
# description: Test medium storage with string compression(> vector size)
|
||||
# group: [string]
|
||||
|
||||
foreach compression fsst dictionary dict_fsst
|
||||
|
||||
statement ok
|
||||
SET storage_compatibility_version='v1.0.0'
|
||||
|
||||
onlyif compression=dict_fsst
|
||||
statement ok
|
||||
SET storage_compatibility_version='v1.3.0'
|
||||
|
||||
load __TEST_DIR__/test_medium_${compression}.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='${compression}'
|
||||
|
||||
# simple dictionary compressed column with few values
|
||||
statement ok
|
||||
CREATE TABLE test (a VARCHAR);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT (i%500)::VARCHAR FROM range(0, 10000) tbl(i);
|
||||
|
||||
query IIII
|
||||
SELECT SUM(a::INT), MIN(a::INT), MAX(a::INT), COUNT(*) FROM test
|
||||
----
|
||||
2495000 0 499 10000
|
||||
|
||||
restart
|
||||
|
||||
query IIII
|
||||
SELECT SUM(a::INT), MIN(a::INT), MAX(a::INT), COUNT(*) FROM test
|
||||
----
|
||||
2495000 0 499 10000
|
||||
|
||||
statement ok
|
||||
CHECKPOINT
|
||||
|
||||
query I
|
||||
SELECT lower(compression)='${compression}' FROM pragma_storage_info('test') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
|
||||
----
|
||||
1
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
endloop
|
||||
93
external/duckdb/test/sql/storage/compression/string/null.test_slow
vendored
Normal file
93
external/duckdb/test/sql/storage/compression/string/null.test_slow
vendored
Normal file
@@ -0,0 +1,93 @@
|
||||
# name: test/sql/storage/compression/string/null.test_slow
|
||||
# description: Test storage of string columns with compression and many null values
|
||||
# group: [string]
|
||||
|
||||
statement ok
|
||||
pragma enable_verification
|
||||
|
||||
foreach compression fsst dictionary dict_fsst
|
||||
|
||||
statement ok
|
||||
SET storage_compatibility_version='v1.0.0'
|
||||
|
||||
onlyif compression=dict_fsst
|
||||
statement ok
|
||||
SET storage_compatibility_version='v1.3.0'
|
||||
|
||||
load __TEST_DIR__/test_null_${compression}.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='${compression}'
|
||||
|
||||
# single NULL value
|
||||
statement ok
|
||||
CREATE or replace TABLE nulls(i VARCHAR)
|
||||
|
||||
statement ok
|
||||
INSERT INTO nulls VALUES (NULL)
|
||||
|
||||
query I
|
||||
SELECT * FROM nulls
|
||||
----
|
||||
NULL
|
||||
|
||||
restart
|
||||
|
||||
query I
|
||||
SELECT * FROM nulls
|
||||
----
|
||||
NULL
|
||||
|
||||
# many null values
|
||||
statement ok
|
||||
CREATE or replace TABLE nulls(i VARCHAR)
|
||||
|
||||
statement ok
|
||||
INSERT INTO nulls SELECT NULL FROM range(70000)
|
||||
|
||||
query III
|
||||
SELECT COUNT(*), COUNT(i::INT), SUM(i::INT) FROM nulls
|
||||
----
|
||||
70000 0 NULL
|
||||
|
||||
restart
|
||||
|
||||
query III
|
||||
SELECT COUNT(*), COUNT(i::INT), SUM(i::INT) FROM nulls
|
||||
----
|
||||
70000 0 NULL
|
||||
|
||||
statement ok
|
||||
select compression from pragma_storage_info('nulls') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1;
|
||||
|
||||
query I
|
||||
select current_setting('force_compression') ILIKE '${compression}'
|
||||
----
|
||||
true
|
||||
|
||||
# mix with non-null values
|
||||
statement ok
|
||||
CREATE OR REPLACE TABLE nulls AS FROM nulls UNION ALL VALUES (1), (1), (1), (2), (2), (2)
|
||||
|
||||
query III
|
||||
SELECT COUNT(*), COUNT(i::INT), SUM(i::INT) FROM nulls
|
||||
----
|
||||
70006 6 9
|
||||
|
||||
restart
|
||||
|
||||
query III
|
||||
SELECT COUNT(*), COUNT(i::INT), SUM(i::INT) FROM nulls
|
||||
----
|
||||
70006 6 9
|
||||
|
||||
statement ok
|
||||
CHECKPOINT
|
||||
|
||||
query I
|
||||
SELECT lower(compression)='${compression}' FROM pragma_storage_info('nulls') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
|
||||
----
|
||||
1
|
||||
|
||||
# compression
|
||||
endloop
|
||||
64
external/duckdb/test/sql/storage/compression/string/null_large.test_slow
vendored
Normal file
64
external/duckdb/test/sql/storage/compression/string/null_large.test_slow
vendored
Normal file
@@ -0,0 +1,64 @@
|
||||
# name: test/sql/storage/compression/string/null_large.test_slow
|
||||
# description: Test storage of string columns with a lot of nulls or empty strings
|
||||
# group: [string]
|
||||
|
||||
load __TEST_DIR__/test_string_compression.db
|
||||
|
||||
foreach compression fsst dictionary dict_fsst
|
||||
|
||||
foreach enable_fsst_vector true false
|
||||
|
||||
statement ok
|
||||
SET enable_fsst_vectors='${enable_fsst_vector}'
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='${compression}'
|
||||
|
||||
# varchars
|
||||
statement ok
|
||||
CREATE TABLE varchars AS SELECT case when i%2=0 then null else concat('thisismyvarchar-', i//4) end AS v FROM range(1000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint;
|
||||
|
||||
query IIIII
|
||||
SELECT MIN(v), MAX(v), COUNT(*), COUNT(v), COUNT(DISTINCT v) FROM varchars
|
||||
----
|
||||
thisismyvarchar-0 thisismyvarchar-99999 1000000 500000 250000
|
||||
|
||||
statement ok
|
||||
DROP TABLE varchars
|
||||
|
||||
endloop
|
||||
|
||||
endloop
|
||||
|
||||
# Do same for empty strings
|
||||
foreach compression fsst dictionary dict_fsst
|
||||
|
||||
foreach enable_fsst_vector true false
|
||||
|
||||
statement ok
|
||||
SET enable_fsst_vectors='${enable_fsst_vector}'
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='${compression}'
|
||||
|
||||
# varchars
|
||||
statement ok
|
||||
CREATE TABLE varchars AS SELECT case when i%2=0 then '' else concat('this-', i//4) end AS v FROM range(1000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint;
|
||||
|
||||
query IIIII
|
||||
SELECT MIN(v), MAX(v), COUNT(*), COUNT(v), COUNT(DISTINCT v) FROM varchars
|
||||
----
|
||||
(empty) this-99999 1000000 1000000 250001
|
||||
|
||||
statement ok
|
||||
DROP TABLE varchars
|
||||
|
||||
endloop
|
||||
|
||||
endloop
|
||||
60
external/duckdb/test/sql/storage/compression/string/simple.test
vendored
Normal file
60
external/duckdb/test/sql/storage/compression/string/simple.test
vendored
Normal file
@@ -0,0 +1,60 @@
|
||||
# name: test/sql/storage/compression/string/simple.test
|
||||
# description: Test dictionary compression
|
||||
# group: [string]
|
||||
|
||||
statement ok
|
||||
ATTACH '__TEST_DIR__/test_blob_new.db' AS db_v13 (STORAGE_VERSION 'v1.3.0');
|
||||
|
||||
statement ok
|
||||
ATTACH '__TEST_DIR__/test_blob_old.db' AS db_v1 (STORAGE_VERSION 'v1.0.0');
|
||||
|
||||
foreach compression fsst dictionary dict_fsst
|
||||
|
||||
statement ok
|
||||
USE db_v1
|
||||
|
||||
onlyif compression=dict_fsst
|
||||
statement ok
|
||||
USE db_v13
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='${compression}'
|
||||
|
||||
# simple compression with few values
|
||||
statement ok
|
||||
CREATE TABLE test (a VARCHAR);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT CONCAT('A-',(i%5)::VARCHAR) FROM range(0,1025) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select * from test limit 5;
|
||||
----
|
||||
A-0
|
||||
A-1
|
||||
A-2
|
||||
A-3
|
||||
A-4
|
||||
|
||||
query I
|
||||
select a[3] from test limit 5;
|
||||
----
|
||||
0
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
|
||||
query I
|
||||
SELECT lower(compression)='${compression}' FROM pragma_storage_info('test') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
|
||||
----
|
||||
1
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
endloop
|
||||
|
||||
51
external/duckdb/test/sql/storage/compression/string/struct.test
vendored
Normal file
51
external/duckdb/test/sql/storage/compression/string/struct.test
vendored
Normal file
@@ -0,0 +1,51 @@
|
||||
# name: test/sql/storage/compression/string/struct.test
|
||||
# description: Test storage with string compression inside structs
|
||||
# group: [string]
|
||||
|
||||
foreach compression fsst dictionary dict_fsst
|
||||
|
||||
statement ok
|
||||
SET storage_compatibility_version='v1.0.0'
|
||||
|
||||
onlyif compression=dict_fsst
|
||||
statement ok
|
||||
SET storage_compatibility_version='v1.3.0'
|
||||
|
||||
load __TEST_DIR__/test_struct_${compression}.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='${compression}'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test (s ROW(a VARCHAR));
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT {'a': concat('0000000000000000000', (i%250)::VARCHAR)} FROM range(0, 10000) tbl(i);
|
||||
INSERT INTO test SELECT {'a': concat('0000000000000000000', (i%250)::VARCHAR)} FROM range(22767, 32767) tbl(i);
|
||||
INSERT INTO test SELECT {'a': concat('0000000000000000000', (i%250)::VARCHAR)} FROM range(2147473647, 2147483647) tbl(i);
|
||||
INSERT INTO test SELECT {'a': concat('0000000000000000000', (i%250)::VARCHAR)} FROM range(2147473647, 2147483647) tbl(i);
|
||||
|
||||
query IIII
|
||||
SELECT SUM(s['a']::INT), MIN(s['a']::INT), MAX(s['a']::INT), COUNT(*) FROM test
|
||||
----
|
||||
4980000 0 249 40000
|
||||
|
||||
restart
|
||||
|
||||
query IIII
|
||||
SELECT SUM(s['a']::INT), MIN(s['a']::INT), MAX(s['a']::INT), COUNT(*) FROM test
|
||||
----
|
||||
4980000 0 249 40000
|
||||
|
||||
statement ok
|
||||
CHECKPOINT
|
||||
|
||||
query I
|
||||
SELECT lower(compression)='${compression}' FROM pragma_storage_info('test') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
|
||||
----
|
||||
1
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
endloop
|
||||
64
external/duckdb/test/sql/storage/compression/string/table_copy.test
vendored
Normal file
64
external/duckdb/test/sql/storage/compression/string/table_copy.test
vendored
Normal file
@@ -0,0 +1,64 @@
|
||||
# name: test/sql/storage/compression/string/table_copy.test
|
||||
# description: Tests a table copy on a table spanning multiple segments
|
||||
# group: [string]
|
||||
|
||||
require no_latest_storage
|
||||
|
||||
require tpch
|
||||
|
||||
foreach compression fsst dictionary dict_fsst
|
||||
|
||||
statement ok
|
||||
SET storage_compatibility_version='v1.0.0'
|
||||
|
||||
onlyif compression=dict_fsst
|
||||
statement ok
|
||||
SET storage_compatibility_version='v1.3.0'
|
||||
|
||||
load __TEST_DIR__/test_table_copy_${compression}.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='${compression}'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test (a VARCHAR);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT (i%500)::VARCHAR FROM range(0,150000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_2 AS SELECT a FROM test;
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select sum(a::INT) from test;
|
||||
----
|
||||
37425000
|
||||
|
||||
query I
|
||||
select sum(a::INT) from test_2;
|
||||
----
|
||||
37425000
|
||||
|
||||
query I
|
||||
SELECT lower(compression)='${compression}' FROM pragma_storage_info('test') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
|
||||
----
|
||||
1
|
||||
|
||||
query I
|
||||
SELECT lower(compression)='${compression}' FROM pragma_storage_info('test_2') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
|
||||
----
|
||||
1
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
statement ok
|
||||
DROP TABLE test_2;
|
||||
|
||||
endloop
|
||||
Reference in New Issue
Block a user