should be it
This commit is contained in:
119
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_bitwidths.test_slow
vendored
Normal file
119
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_bitwidths.test_slow
vendored
Normal file
@@ -0,0 +1,119 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_bitwidths.test_slow
|
||||
# description: Test bitpacking with values that compress to all different widths
|
||||
# group: [bitpacking]
|
||||
|
||||
# load the DB from disk
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression = 'bitpacking'
|
||||
|
||||
foreach bitpacking_mode delta_for for constant_delta constant
|
||||
|
||||
foreach typesize 8 16 32 64
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='${bitpacking_mode}'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_unsigned AS SELECT cast(2**(i//2048) as UINT${typesize}) i FROM range(${typesize}*2048) tbl(i);
|
||||
|
||||
statement ok
|
||||
CHECKPOINT;
|
||||
|
||||
query I
|
||||
SELECT count(*)=${typesize} FROM (SELECT count(i) FROM test_unsigned GROUP BY i);
|
||||
----
|
||||
1
|
||||
|
||||
query I
|
||||
SELECT DISTINCT(c_i) FROM (SELECT count(i) AS c_i FROM test_unsigned GROUP BY i);
|
||||
----
|
||||
2048
|
||||
|
||||
query IIII
|
||||
select count(*)=1, count(*)=(${typesize}-1) , AVG(c_i), (i//delta) AS diff_to_next_row from (
|
||||
SELECT i, count(i) as c_i, lag(i, 1) OVER (ORDER BY i) delta FROM test_unsigned GROUP BY i
|
||||
) GROUP BY diff_to_next_row ORDER BY ALL;
|
||||
----
|
||||
False True 2048.000000 2
|
||||
True False 2048.000000 NULL
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_signed_neg AS SELECT cast(-(2**(i//2048)) as INT${typesize}) i FROM range(${typesize}*2048) tbl(i);
|
||||
|
||||
statement ok
|
||||
CHECKPOINT;
|
||||
|
||||
query I
|
||||
SELECT count(*)=${typesize} FROM (SELECT count(i) FROM test_signed_neg GROUP BY i);
|
||||
----
|
||||
1
|
||||
|
||||
query I
|
||||
SELECT DISTINCT(c_i) FROM (SELECT count(i) AS c_i FROM test_signed_neg GROUP BY i);
|
||||
----
|
||||
2048
|
||||
|
||||
query IIII
|
||||
select count(*)=1, count(*)=(${typesize}-1) , AVG(c_i), (i//delta) AS diff_to_next_row from (
|
||||
SELECT i, count(i) as c_i, lag(i, 1) OVER (ORDER BY i) delta FROM test_unsigned GROUP BY i
|
||||
) GROUP BY diff_to_next_row ORDER BY ALL;
|
||||
----
|
||||
False True 2048.000000 2
|
||||
True False 2048.000000 NULL
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_signed_pos AS SELECT cast(2**(i//2048) as INT${typesize}) i FROM range((${typesize}-1)*2048) tbl(i);
|
||||
|
||||
statement ok
|
||||
CHECKPOINT;
|
||||
|
||||
query I
|
||||
SELECT count(*)=(${typesize}-1) FROM (SELECT count(i) FROM test_signed_pos GROUP BY i);
|
||||
----
|
||||
1
|
||||
|
||||
query I
|
||||
SELECT DISTINCT(c_i) FROM (SELECT count(i) AS c_i FROM test_signed_neg GROUP BY i);
|
||||
----
|
||||
2048
|
||||
|
||||
query IIII
|
||||
select count(*)=1, count(*)=(${typesize}-1) , AVG(c_i), (i//delta) AS diff_to_next_row from (
|
||||
SELECT i, count(i) as c_i, lag(i, 1) OVER (ORDER BY i) delta FROM test_unsigned GROUP BY i
|
||||
) GROUP BY diff_to_next_row ORDER BY ALL;
|
||||
----
|
||||
False True 2048.000000 2
|
||||
True False 2048.000000 NULL
|
||||
|
||||
statement ok
|
||||
DROP TABLE test_unsigned
|
||||
|
||||
statement ok
|
||||
DROP TABLE test_signed_neg
|
||||
|
||||
statement ok
|
||||
DROP TABLE test_signed_pos
|
||||
|
||||
endloop
|
||||
|
||||
foreach type <integral> bool
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_nullpack AS SELECT CAST((i//3000)%2 as ${type}) as i FROM range(0,12000) tbl(i);
|
||||
|
||||
statement ok
|
||||
CHECKPOINT;
|
||||
|
||||
query I
|
||||
SELECT AVG(cast (i as int)) FROM test_nullpack
|
||||
----
|
||||
0.5
|
||||
|
||||
statement ok
|
||||
drop table test_nullpack
|
||||
|
||||
endloop
|
||||
|
||||
endloop
|
||||
253
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_compression_ratio.test_slow
vendored
Normal file
253
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_compression_ratio.test_slow
vendored
Normal file
@@ -0,0 +1,253 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_compression_ratio.test_slow
|
||||
# description: Assert bitpacking compression ratio is within reasonable margins for each mode
|
||||
# group: [bitpacking]
|
||||
|
||||
# This test defaults to another compression function for smaller block sizes,
|
||||
# because the bitpacking groups no longer fit the blocks.
|
||||
require block_size 262144
|
||||
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
#### CONSTANT MODE Compression ratio calculation:
|
||||
# For single row group (note we choose values such that we don't create the constant segments):
|
||||
# 59 vectors with CONSTANT mode, the last one will be FOR mode
|
||||
# Total compressed bytes = 59*(8+4) + 1*(2048/8 + 8 + 8 + 4) = 984
|
||||
# Total uncompressed bytes = 120000*8 = 960000
|
||||
# Ratio ~= 975x
|
||||
# However, because this completely fills up a block and we do not support block sharing yet, we waste a lot of space
|
||||
statement ok
|
||||
PRAGMA force_compression='bitpacking'
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='constant'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_bitpacked AS SELECT (i//119000::INT64)::INT64 AS i FROM range(0, 120000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='uncompressed'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_uncompressed AS SELECT i::INT64 FROM range(0, 120000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_bitpacked') WHERE segment_type != 'VALIDITY' AND compression != 'BitPacking';
|
||||
----
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_uncompressed') WHERE segment_type != 'VALIDITY' AND compression != 'Uncompressed';
|
||||
----
|
||||
|
||||
query II
|
||||
select (uncompressed::FLOAT / bitpacked::FLOAT) > 700, (uncompressed::FLOAT / bitpacked::FLOAT) < 1000 FROM (
|
||||
select
|
||||
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
|
||||
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
|
||||
)
|
||||
----
|
||||
True True
|
||||
|
||||
statement ok
|
||||
drop table test_bitpacked;
|
||||
drop table test_uncompressed;
|
||||
|
||||
#### CONSTANT DELTA MODE Compression ratio calculation:
|
||||
# For single row group
|
||||
# 60 vectors with a constant increase (1)
|
||||
# Total compressed bytes = 60*(8+8+4) = 1200
|
||||
# Total uncompressed bytes = 120000*8 = 960000
|
||||
# Expected Ratio ~= 800x
|
||||
statement ok
|
||||
PRAGMA force_compression='bitpacking'
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='constant_delta'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_bitpacked AS SELECT i::INT64 AS i FROM range(0, 120000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='uncompressed'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_uncompressed AS SELECT i::INT64 AS i FROM range(0, 120000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_bitpacked') WHERE segment_type != 'VALIDITY' AND compression != 'BitPacking';
|
||||
----
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_uncompressed') WHERE segment_type != 'VALIDITY' AND compression != 'Uncompressed';
|
||||
----
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query II
|
||||
select (uncompressed::FLOAT / bitpacked::FLOAT) > 600, (uncompressed::FLOAT / bitpacked::FLOAT) < 800 FROM (
|
||||
select
|
||||
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
|
||||
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
|
||||
)
|
||||
----
|
||||
True True
|
||||
|
||||
statement ok
|
||||
drop table test_bitpacked;
|
||||
drop table test_uncompressed;
|
||||
|
||||
#### DELTA FOR MODE Compression ratio calculation:
|
||||
# For single row group
|
||||
# 60 vectors with DELTA_FOR mode smallest possible compression
|
||||
# Total compressed bytes = 60*(8+8+4+(2048/8)) = 16560
|
||||
# Total uncompressed bytes = 120000*8 = 960000
|
||||
# Expected Ratio ~= 58x
|
||||
statement ok
|
||||
PRAGMA force_compression='bitpacking'
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='delta_for'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_bitpacked AS SELECT i//2::INT64 AS i FROM range(0, 120000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='uncompressed'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_uncompressed AS SELECT i AS i FROM range(0, 120000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_bitpacked') WHERE segment_type != 'VALIDITY' AND compression != 'BitPacking';
|
||||
----
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_uncompressed') WHERE segment_type != 'VALIDITY' AND compression != 'Uncompressed';
|
||||
----
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query II
|
||||
select (uncompressed::FLOAT / bitpacked::FLOAT) > 50, (uncompressed::FLOAT / bitpacked::FLOAT) < 60 FROM (
|
||||
select
|
||||
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
|
||||
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
|
||||
)
|
||||
----
|
||||
True True
|
||||
|
||||
statement ok
|
||||
drop table test_bitpacked;
|
||||
drop table test_uncompressed;
|
||||
|
||||
# FOR MODE Compression ratio calculation:
|
||||
# For single row group
|
||||
# 60 vectors with DELTA_FOR mode smallest possible compression
|
||||
# Total compressed bytes = 60*(8+4+(2048/8)) = 16080
|
||||
# Total uncompressed bytes = 120000*8 = 960000
|
||||
# Expected Ratio ~= 60x
|
||||
statement ok
|
||||
PRAGMA force_compression='bitpacking'
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='for'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_bitpacked AS SELECT i%2::INT64 AS i FROM range(0, 120000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='uncompressed'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_uncompressed AS SELECT i::INT64 AS i FROM range(0, 120000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_bitpacked') WHERE segment_type != 'VALIDITY' AND compression != 'BitPacking';
|
||||
----
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_uncompressed') WHERE segment_type != 'VALIDITY' AND compression != 'Uncompressed';
|
||||
----
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query II
|
||||
select (uncompressed::FLOAT / bitpacked::FLOAT) > 50, (uncompressed::FLOAT / bitpacked::FLOAT) < 60 FROM (
|
||||
select
|
||||
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
|
||||
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
|
||||
)
|
||||
----
|
||||
True True
|
||||
|
||||
statement ok
|
||||
drop table test_bitpacked;
|
||||
drop table test_uncompressed;
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='none'
|
||||
|
||||
# Assert that all supported types do in fact compress
|
||||
foreach type int8 int16 int32 int64 uint8 uint16 uint32 uint64 decimal(4,1) decimal(8,1) decimal(12,1) decimal(18,1) bool
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='uncompressed';
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_uncompressed AS SELECT (i%2)::${type} FROM range(0, 2500000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='bitpacking'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_bitpacked AS SELECT (i%2)::${type} FROM range(0, 2500000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
# assert compression ratio >2 wich should be achieved for even the smallest types for this data
|
||||
query II
|
||||
select (uncompressed::FLOAT / bitpacked::FLOAT) > 2, CAST(1 as ${type}) FROM (
|
||||
select
|
||||
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
|
||||
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
|
||||
)
|
||||
----
|
||||
1 1
|
||||
|
||||
statement ok
|
||||
drop table test_bitpacked
|
||||
|
||||
statement ok
|
||||
drop table test_uncompressed
|
||||
|
||||
endloop
|
||||
@@ -0,0 +1,194 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_compression_ratio_hugeint.test_slow
|
||||
# description: Assert bitpacking compression ratio is within reasonable margins for each mode
|
||||
# group: [bitpacking]
|
||||
|
||||
# This test defaults to another compression function for smaller block sizes,
|
||||
# because the bitpacking groups no longer fit the blocks.
|
||||
require block_size 262144
|
||||
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
#### CONSTANT MODE:
|
||||
# Ratio ~= 1000x
|
||||
statement ok
|
||||
PRAGMA force_compression='bitpacking'
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='constant'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_bitpacked AS SELECT (i//119000::INT64)::HUGEINT AS i FROM range(0, 120000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='uncompressed'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_uncompressed AS SELECT i::HUGEINT FROM range(0, 120000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_bitpacked') WHERE segment_type != 'VALIDITY' AND compression != 'BitPacking';
|
||||
----
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_uncompressed') WHERE segment_type != 'VALIDITY' AND compression != 'Uncompressed';
|
||||
----
|
||||
|
||||
query II
|
||||
select (uncompressed::FLOAT / bitpacked::FLOAT) > 900, (uncompressed::FLOAT / bitpacked::FLOAT) < 1200 FROM (
|
||||
select
|
||||
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
|
||||
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
|
||||
)
|
||||
----
|
||||
True True
|
||||
|
||||
statement ok
|
||||
drop table test_bitpacked;
|
||||
drop table test_uncompressed;
|
||||
|
||||
#### CONSTANT DELTA MODE:
|
||||
# Expected Ratio ~= 800x
|
||||
statement ok
|
||||
PRAGMA force_compression='bitpacking'
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='constant_delta'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_bitpacked AS SELECT i::HUGEINT AS i FROM range(0, 120000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='uncompressed'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_uncompressed AS SELECT i::HUGEINT AS i FROM range(0, 120000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_bitpacked') WHERE segment_type != 'VALIDITY' AND compression != 'BitPacking';
|
||||
----
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_uncompressed') WHERE segment_type != 'VALIDITY' AND compression != 'Uncompressed';
|
||||
----
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query II
|
||||
select (uncompressed::FLOAT / bitpacked::FLOAT) > 600, (uncompressed::FLOAT / bitpacked::FLOAT) < 800 FROM (
|
||||
select
|
||||
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
|
||||
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
|
||||
)
|
||||
----
|
||||
True True
|
||||
|
||||
statement ok
|
||||
drop table test_bitpacked;
|
||||
drop table test_uncompressed;
|
||||
|
||||
#### DELTA FOR MODE:
|
||||
# Expected Ratio ~= 50x
|
||||
statement ok
|
||||
PRAGMA force_compression='bitpacking'
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='delta_for'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_bitpacked AS SELECT i//2::HUGEINT AS i FROM range(0, 120000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='uncompressed'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_uncompressed AS SELECT i AS i FROM range(0, 120000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_bitpacked') WHERE segment_type != 'VALIDITY' AND compression != 'BitPacking';
|
||||
----
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_uncompressed') WHERE segment_type != 'VALIDITY' AND compression != 'Uncompressed';
|
||||
----
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query II
|
||||
select (uncompressed::FLOAT / bitpacked::FLOAT) > 40, (uncompressed::FLOAT / bitpacked::FLOAT) < 60 FROM (
|
||||
select
|
||||
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
|
||||
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
|
||||
)
|
||||
----
|
||||
True True
|
||||
|
||||
statement ok
|
||||
drop table test_bitpacked;
|
||||
drop table test_uncompressed;
|
||||
|
||||
# FOR MODE:
|
||||
# Expected Ratio ~= 95x
|
||||
statement ok
|
||||
PRAGMA force_compression='bitpacking'
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='for'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_bitpacked AS SELECT i%2::HUGEINT AS i FROM range(0, 120000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='uncompressed'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_uncompressed AS SELECT i::HUGEINT AS i FROM range(0, 120000000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_bitpacked') WHERE segment_type != 'VALIDITY' AND compression != 'BitPacking';
|
||||
----
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_uncompressed') WHERE segment_type != 'VALIDITY' AND compression != 'Uncompressed';
|
||||
----
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query II
|
||||
select (uncompressed::FLOAT / bitpacked::FLOAT) > 90, (uncompressed::FLOAT / bitpacked::FLOAT) < 100 FROM (
|
||||
select
|
||||
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
|
||||
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
|
||||
)
|
||||
----
|
||||
True True
|
||||
|
||||
statement ok
|
||||
drop table test_bitpacked;
|
||||
drop table test_uncompressed;
|
||||
69
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_constant_delta.test
vendored
Normal file
69
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_constant_delta.test
vendored
Normal file
@@ -0,0 +1,69 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_constant_delta.test
|
||||
# description: Test that will use the BitpackingMode::CONSTANT_DELTA compression mode
|
||||
# group: [bitpacking]
|
||||
|
||||
# This test defaults to another compression function for smaller block sizes,
|
||||
# because the bitpacking groups no longer fit the blocks.
|
||||
require block_size 262144
|
||||
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression = 'bitpacking'
|
||||
|
||||
foreach bitpacking_mode delta_for for constant_delta constant
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='${bitpacking_mode}'
|
||||
|
||||
foreach type int8 int16 int32 int64 uint8 uint16 uint32 uint64 hugeint decimal(4,1) decimal(8,1) decimal(12,1) decimal(18,1)
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test (c ${type});
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT 2+i*2::${type} FROM range(0,5) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') where segment_type != 'VALIDITY' and compression != 'BitPacking'
|
||||
----
|
||||
|
||||
query I
|
||||
SELECT * FROM test;
|
||||
----
|
||||
2
|
||||
4
|
||||
6
|
||||
8
|
||||
10
|
||||
|
||||
statement ok
|
||||
DROP TABLE test
|
||||
|
||||
endloop
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test (c INT64);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i from range(0,130000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') where segment_type != 'VALIDITY' and compression != 'BitPacking'
|
||||
----
|
||||
|
||||
query I
|
||||
SELECT avg(c) FROM test;
|
||||
----
|
||||
64999.5
|
||||
|
||||
statement ok
|
||||
DROP TABLE test
|
||||
|
||||
endloop
|
||||
68
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_delta.test_slow
vendored
Normal file
68
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_delta.test_slow
vendored
Normal file
@@ -0,0 +1,68 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_delta.test_slow
|
||||
# description: Test some large incompressible data
|
||||
# group: [bitpacking]
|
||||
|
||||
# This test defaults to another compression function for smaller block sizes,
|
||||
# because the bitpacking groups no longer fit the blocks.
|
||||
require block_size 262144
|
||||
|
||||
load __TEST_DIR__/bitpacking_uncompressible.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='bitpacking'
|
||||
|
||||
foreach bitpacking_mode delta_for for constant_delta constant
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='${bitpacking_mode}'
|
||||
|
||||
# simple compression with few values
|
||||
statement ok
|
||||
CREATE TABLE test_delta_full_range (a UINT64);
|
||||
|
||||
# Insert multiple ranges so that each method can be used on at least on the the ranges
|
||||
statement ok
|
||||
INSERT INTO test_delta_full_range select case when i%2=0 then 0 else 18446744073709551615 end from range(0,1000000) tbl(i);
|
||||
|
||||
query II
|
||||
select a, count(*) from test_delta_full_range group by a order by a;
|
||||
----
|
||||
0 500000
|
||||
18446744073709551615 500000
|
||||
|
||||
query I
|
||||
SELECT DISTINCT compression FROM pragma_storage_info('test_delta_full_range') where segment_type = 'UBIGINT'
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
drop table test_delta_full_range
|
||||
|
||||
endloop
|
||||
|
||||
# Do the same thing and confirm we don't bitpack here
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='none'
|
||||
|
||||
# simple compression with few values
|
||||
statement ok
|
||||
CREATE TABLE test_delta_full_range (a UINT64);
|
||||
|
||||
# Insert multiple ranges so that each method can be used on at least on the the ranges
|
||||
statement ok
|
||||
INSERT INTO test_delta_full_range select case when i%2=0 then 0 else 18446744073709551615 end from range(0,1000000) tbl(i);
|
||||
|
||||
query II
|
||||
select a, count(*) from test_delta_full_range group by a order by a;
|
||||
----
|
||||
0 500000
|
||||
18446744073709551615 500000
|
||||
|
||||
query I
|
||||
SELECT DISTINCT compression FROM pragma_storage_info('test_delta_full_range') where segment_type = 'UBIGINT'
|
||||
----
|
||||
Uncompressed
|
||||
|
||||
statement ok
|
||||
drop table test_delta_full_range
|
||||
26
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_delta_for.test
vendored
Normal file
26
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_delta_for.test
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_delta_for.test
|
||||
# description: Test bitpacking delta for
|
||||
# group: [bitpacking]
|
||||
|
||||
# This test defaults to another compression function for smaller block sizes,
|
||||
# because the bitpacking groups no longer fit the blocks.
|
||||
require block_size 262144
|
||||
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='bitpacking'
|
||||
|
||||
statement ok
|
||||
create table aux as select range::INT x from range(-2_000_000_000, 2_000_000_000, 2_000_000);
|
||||
|
||||
statement ok
|
||||
create table tt as select (x + if (random() > 0.5, 1, -1)) x from aux;
|
||||
|
||||
statement ok
|
||||
CHECKPOINT;
|
||||
|
||||
query I
|
||||
select compression from pragma_storage_info('tt') where segment_type != 'VALIDITY';
|
||||
----
|
||||
BitPacking
|
||||
42
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_fatal_forced.test_slow
vendored
Normal file
42
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_fatal_forced.test_slow
vendored
Normal file
@@ -0,0 +1,42 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_fatal_forced.test_slow
|
||||
# description: Test forced bitpacking, with value ranges that are rejected by the bitpacking compression analyze step
|
||||
# group: [bitpacking]
|
||||
|
||||
# load the DB from disk
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
foreach bitpacking_mode delta_for for constant_delta constant
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='${bitpacking_mode}'
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression = 'bitpacking'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test (x INT128, a INT64, b INT32, c INT16, d TINYINT);
|
||||
|
||||
# Data too big to be compressed with bitpacking
|
||||
statement ok
|
||||
INSERT INTO test VALUES
|
||||
(-170141183460469231731687303715884105728, -9223372036854775808, -2147483648, -32768, -128),
|
||||
(170141183460469231731687303715884105727, 9223372036854775807, 2147483647, 32767, 127);
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE (
|
||||
segment_type ILIKE 'HUGEINT' OR
|
||||
segment_type ILIKE 'BIGINT' OR
|
||||
segment_type ILIKE 'INTEGER' OR
|
||||
segment_type ILIKE 'SMALLINT' OR
|
||||
segment_type ILIKE 'TINYINT')
|
||||
----
|
||||
Uncompressed
|
||||
Uncompressed
|
||||
Uncompressed
|
||||
Uncompressed
|
||||
Uncompressed
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
endloop
|
||||
60
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_filter_pushdown.test
vendored
Normal file
60
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_filter_pushdown.test
vendored
Normal file
@@ -0,0 +1,60 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_filter_pushdown.test
|
||||
# description: Filter pushdown with Bitpacking columns
|
||||
# group: [bitpacking]
|
||||
|
||||
# This test defaults to another compression function for smaller block sizes,
|
||||
# because the bitpacking groups no longer fit the blocks.
|
||||
require block_size 262144
|
||||
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression = 'bitpacking'
|
||||
|
||||
foreach bitpacking_mode auto delta_for for constant_delta constant
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='${bitpacking_mode}'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, col INTEGER)
|
||||
|
||||
# Insert various data to ensure theres something compressible for all bitpacking modes
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR id, i b FROM range(10000) tbl(i)
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR id, 1337 FROM range(20000, 30000) tbl(i)
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR id, i b FROM range(30000,40000) tbl(i)
|
||||
|
||||
statement ok
|
||||
CHECKPOINT
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') where segment_type = 'INTEGER' and compression != 'BitPacking'
|
||||
----
|
||||
|
||||
# filter on the bitpacking column
|
||||
query IIII
|
||||
SELECT SUM(col), MIN(col), MAX(col), COUNT(*) FROM test WHERE col=1337
|
||||
----
|
||||
13371337 1337 1337 10001
|
||||
|
||||
# filter on non-bitpacking column
|
||||
query IIIIII
|
||||
SELECT MIN(id), MAX(id), SUM(col), MIN(col), MAX(col), COUNT(*) FROM test WHERE id='5000'
|
||||
----
|
||||
5000 5000 5000 5000 5000 1
|
||||
|
||||
# filter on non-bitpacking column
|
||||
query IIIIII
|
||||
SELECT MIN(id), MAX(id), SUM(col), MIN(col), MAX(col), COUNT(*) FROM test WHERE id::INT64%1000=0;
|
||||
----
|
||||
0 9000 403370 0 39000 30
|
||||
|
||||
statement ok
|
||||
DROP TABLE test
|
||||
|
||||
endloop
|
||||
255
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_hugeint.test
vendored
Normal file
255
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_hugeint.test
vendored
Normal file
@@ -0,0 +1,255 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_hugeint.test
|
||||
# description: Test hugeint bitpacking at multiple bitwidths
|
||||
# group: [bitpacking]
|
||||
|
||||
# This test defaults to another compression function for smaller block sizes,
|
||||
# because the bitpacking groups no longer fit the blocks.
|
||||
require block_size 262144
|
||||
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='bitpacking'
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='constant'
|
||||
|
||||
# bit-width < 32
|
||||
# ----------------------------------------
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a HUGEINT);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, i::HUGEINT FROM range(0, 16) tbl(i);
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0, 16) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
0
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
|
||||
# make sure compression is being used
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'HUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
# bit-width == 32
|
||||
# ----------------------------------------
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a HUGEINT);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, i::HUGEINT + 3000000000 FROM range(0, 16) tbl(i);
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0, 16) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
3000000000
|
||||
3000000001
|
||||
3000000002
|
||||
3000000003
|
||||
3000000004
|
||||
|
||||
# make sure compression is being used
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'HUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
# 32 < bit-width < 64
|
||||
# ----------------------------------------
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a HUGEINT);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, i::HUGEINT + 200000000000 FROM range(0, 16) tbl(i);
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0, 16) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
200000000000
|
||||
200000000001
|
||||
200000000002
|
||||
200000000003
|
||||
200000000004
|
||||
|
||||
# make sure compression is being used
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'HUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
# bit-width == 64
|
||||
# ----------------------------------------
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a HUGEINT);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, i::HUGEINT + 10000000000000000000 FROM range(0, 16) tbl(i);
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0, 16) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
10000000000000000000
|
||||
10000000000000000001
|
||||
10000000000000000002
|
||||
10000000000000000003
|
||||
10000000000000000004
|
||||
|
||||
# make sure compression is being used
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'HUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
# 64 < bit-width < 96
|
||||
# ----------------------------------------
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a HUGEINT);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, i::HUGEINT + 500000000000000000000 FROM range(0, 16) tbl(i);
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0, 16) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
500000000000000000000
|
||||
500000000000000000001
|
||||
500000000000000000002
|
||||
500000000000000000003
|
||||
500000000000000000004
|
||||
|
||||
# make sure compression is being used
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'HUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
# bit-width == 96
|
||||
# ----------------------------------------
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a HUGEINT);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, i::HUGEINT + 50000000000000000000000000000 FROM range(0, 16) tbl(i);
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0, 16) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
50000000000000000000000000000
|
||||
50000000000000000000000000001
|
||||
50000000000000000000000000002
|
||||
50000000000000000000000000003
|
||||
50000000000000000000000000004
|
||||
|
||||
# make sure compression is being used
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'HUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
# 96 < bit-width < 128
|
||||
# ----------------------------------------
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a HUGEINT);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, i::HUGEINT + 300000000000000000000000000000000 FROM range(0, 16) tbl(i);
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0, 16) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
300000000000000000000000000000000
|
||||
300000000000000000000000000000001
|
||||
300000000000000000000000000000002
|
||||
300000000000000000000000000000003
|
||||
300000000000000000000000000000004
|
||||
|
||||
# make sure compression is being used
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'HUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
# bit-width == 128
|
||||
# ----------------------------------------
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a HUGEINT);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, i::HUGEINT + 20000000000000000000000000000000000000 FROM range(0, 16) tbl(i);
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0, 16) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
20000000000000000000000000000000000000
|
||||
20000000000000000000000000000000000001
|
||||
20000000000000000000000000000000000002
|
||||
20000000000000000000000000000000000003
|
||||
20000000000000000000000000000000000004
|
||||
|
||||
# make sure compression is being used
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'HUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
62
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_index_fetch.test_slow
vendored
Normal file
62
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_index_fetch.test_slow
vendored
Normal file
@@ -0,0 +1,62 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_index_fetch.test_slow
|
||||
# description: Fetch from Bitpacking column with index
|
||||
# group: [bitpacking]
|
||||
|
||||
# This test defaults to another compression function for smaller block sizes,
|
||||
# because the bitpacking groups no longer fit the blocks.
|
||||
require block_size 262144
|
||||
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression = 'bitpacking'
|
||||
|
||||
foreach bitpacking_mode delta_for for constant_delta constant
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='${bitpacking_mode}'
|
||||
|
||||
foreach type INTEGER UINT16
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test(id INTEGER PRIMARY KEY, col ${type})
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR id, i b FROM range(10000) tbl(i)
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR id, 1337 FROM range(10000, 20000) tbl(i)
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR id, i b FROM range(20000, 30000) tbl(i)
|
||||
|
||||
statement ok
|
||||
CHECKPOINT
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'INTEGER' and compression != 'BitPacking'
|
||||
----
|
||||
|
||||
query IIIIII
|
||||
SELECT MIN(id), MAX(id), SUM(col), MIN(col), MAX(col), COUNT(*) FROM test WHERE id='5000'
|
||||
----
|
||||
5000 5000 5000 5000 5000 1
|
||||
|
||||
query IIIIII
|
||||
SELECT MIN(id), MAX(id), SUM(col), MIN(col), MAX(col), COUNT(*) FROM test WHERE id='12000'
|
||||
----
|
||||
12000 12000 1337 1337 1337 1
|
||||
|
||||
query IIIIII
|
||||
SELECT MIN(id), MAX(id), SUM(col), MIN(col), MAX(col), COUNT(*) FROM test WHERE id='22000'
|
||||
----
|
||||
22000 22000 22000 22000 22000 1
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
endloop
|
||||
|
||||
endloop
|
||||
|
||||
|
||||
72
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_lists.test_slow
vendored
Normal file
72
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_lists.test_slow
vendored
Normal file
@@ -0,0 +1,72 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_lists.test_slow
|
||||
# description: Test storage with Bitpacking inside lists
|
||||
# group: [bitpacking]
|
||||
|
||||
# This test defaults to another compression function for smaller block sizes,
|
||||
# because the bitpacking groups no longer fit the blocks.
|
||||
require block_size 262144
|
||||
|
||||
load __TEST_DIR__/test_rle.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression = 'bitpacking'
|
||||
|
||||
foreach bitpacking_mode delta_for for constant_delta constant
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='${bitpacking_mode}'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test (id INTEGER, l INTEGER[]);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i, case when (i//1000)%2=0 then [1, 1, 1] else [2, 2] end FROM range(200000) tbl(i)
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'INTEGER' LIMIT 1
|
||||
----
|
||||
BitPacking
|
||||
|
||||
# full unnest
|
||||
query II
|
||||
SELECT COUNT(*), SUM(i) FROM (SELECT UNNEST(l) FROM test) tbl(i)
|
||||
----
|
||||
500000 700000
|
||||
|
||||
# filters/skips
|
||||
query II
|
||||
SELECT COUNT(*), SUM(i) FROM (SELECT UNNEST(l) FROM test WHERE id>=5000 AND id<6000) tbl(i)
|
||||
----
|
||||
2000 4000
|
||||
|
||||
# zonemaps
|
||||
query II
|
||||
SELECT COUNT(*), SUM(i) FROM (SELECT UNNEST(l) FROM test WHERE id>=150000 AND id<160000) tbl(i)
|
||||
----
|
||||
25000 35000
|
||||
|
||||
statement ok
|
||||
CREATE INDEX i_index ON test(id)
|
||||
|
||||
# index lookup in lists
|
||||
query II
|
||||
SELECT * FROM test WHERE id=150001
|
||||
----
|
||||
150001 [1, 1, 1]
|
||||
|
||||
# large lists
|
||||
statement ok
|
||||
CREATE TABLE test_large_list AS SELECT i%100 AS id, LIST(-i) AS list FROM range(0,100000) tbl(i) GROUP BY id;
|
||||
|
||||
query II
|
||||
SELECT COUNT(*), SUM(i) FROM (SELECT UNNEST(list) FROM test_large_list) tbl(i)
|
||||
----
|
||||
100000 -4999950000
|
||||
|
||||
statement ok
|
||||
drop table test_large_list
|
||||
|
||||
statement ok
|
||||
drop table test
|
||||
|
||||
endloop
|
||||
29
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_mode.test
vendored
Normal file
29
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_mode.test
vendored
Normal file
@@ -0,0 +1,29 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_mode.test
|
||||
# description: Test bitpacking mode
|
||||
# group: [bitpacking]
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression = 'bitpacking'
|
||||
|
||||
query I
|
||||
SELECT current_setting('force_bitpacking_mode')
|
||||
----
|
||||
auto
|
||||
|
||||
statement error
|
||||
PRAGMA force_bitpacking_mode='xxx'
|
||||
----
|
||||
Unrecognized option
|
||||
|
||||
|
||||
foreach mode auto constant constant_delta delta_for for
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='${mode}'
|
||||
|
||||
query I
|
||||
SELECT current_setting('force_bitpacking_mode')='${mode}'
|
||||
----
|
||||
true
|
||||
|
||||
endloop
|
||||
50
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_nulls.test
vendored
Normal file
50
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_nulls.test
vendored
Normal file
@@ -0,0 +1,50 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_nulls.test
|
||||
# description: Test bitpacking with nulls
|
||||
# group: [bitpacking]
|
||||
|
||||
# This test defaults to another compression function for smaller block sizes,
|
||||
# because the bitpacking groups no longer fit the blocks.
|
||||
require block_size 262144
|
||||
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='bitpacking'
|
||||
|
||||
foreach bitpacking_mode delta_for for constant_delta constant
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='${bitpacking_mode}'
|
||||
|
||||
# simple compression with few values
|
||||
statement ok
|
||||
CREATE TABLE test (a BIGINT);
|
||||
|
||||
# Constant compressible range
|
||||
statement ok
|
||||
INSERT INTO test SELECT case when i%5=0 then null else 1337 end FROM range(0,10000) tbl(i);
|
||||
|
||||
# Constant delta compressible range
|
||||
statement ok
|
||||
INSERT INTO test SELECT case when i%5=0 then null else i end FROM range(0,10000) tbl(i);
|
||||
|
||||
# FOR/FOR-delta compressible range
|
||||
statement ok
|
||||
INSERT INTO test SELECT case when i%5=0 then null else i//2 end FROM range(0,10000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'BIGINT' and compression != 'BitPacking';
|
||||
----
|
||||
|
||||
query III
|
||||
select sum(a), min(a), max(a) from test;
|
||||
----
|
||||
70694000 0 9999
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
endloop
|
||||
65
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_simple.test
vendored
Normal file
65
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_simple.test
vendored
Normal file
@@ -0,0 +1,65 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_simple.test
|
||||
# description: Test storage bitpacking, but simple
|
||||
# group: [bitpacking]
|
||||
|
||||
# This test defaults to another compression function for smaller block sizes,
|
||||
# because the bitpacking groups no longer fit the blocks.
|
||||
require block_size 262144
|
||||
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='bitpacking'
|
||||
|
||||
foreach bitpacking_mode delta_for for constant_delta constant
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='${bitpacking_mode}'
|
||||
|
||||
# simple compression with few values
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a BIGINT);
|
||||
|
||||
# insert multiple ranges so that each method can be used on at least one the the ranges
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, -i FROM range(0,10000) tbl(i);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0,10000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
0
|
||||
-1
|
||||
-2
|
||||
-3
|
||||
-4
|
||||
|
||||
query I
|
||||
select a from test limit 5 offset 12000;
|
||||
----
|
||||
13371337
|
||||
13371337
|
||||
13371337
|
||||
13371337
|
||||
13371337
|
||||
|
||||
query I
|
||||
select avg(a) from test;
|
||||
----
|
||||
6683168.75
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'BIGINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
endloop
|
||||
60
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_simple_hugeint.test
vendored
Normal file
60
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_simple_hugeint.test
vendored
Normal file
@@ -0,0 +1,60 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_simple_hugeint.test
|
||||
# description: Test storage bitpacking, but simple and for hugeints
|
||||
# group: [bitpacking]
|
||||
|
||||
# This test defaults to another compression function for smaller block sizes,
|
||||
# because the bitpacking groups no longer fit the blocks.
|
||||
require block_size 262144
|
||||
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='bitpacking'
|
||||
|
||||
foreach bitpacking_mode delta_for for constant_delta constant
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='${bitpacking_mode}'
|
||||
|
||||
# simple compression with few values
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a HUGEINT);
|
||||
|
||||
# insert multiple ranges so that each method can be used on at least on the the ranges bit-width > 96
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, -i::HUGEINT + -1234567891011121314151617180000::HUGEINT FROM range(0, 10000) tbl(i);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0,10000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
-1234567891011121314151617180000
|
||||
-1234567891011121314151617180001
|
||||
-1234567891011121314151617180002
|
||||
-1234567891011121314151617180003
|
||||
-1234567891011121314151617180004
|
||||
|
||||
query I
|
||||
select a from test limit 5 offset 12000;
|
||||
----
|
||||
13371337
|
||||
13371337
|
||||
13371337
|
||||
13371337
|
||||
13371337
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'HUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
endloop
|
||||
20
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_size_calculation.test
vendored
Normal file
20
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_size_calculation.test
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_size_calculation.test
|
||||
# description: Test for a bug found in the size calculation
|
||||
# group: [bitpacking]
|
||||
|
||||
require parquet
|
||||
|
||||
require httpfs
|
||||
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
statement ok
|
||||
pragma force_compression='bitpacking'
|
||||
|
||||
statement ok
|
||||
CREATE OR REPLACE TABLE toy_table AS
|
||||
SELECT *
|
||||
FROM 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/bp_bug.parquet' ;
|
||||
|
||||
statement ok
|
||||
CHECKPOINT;
|
||||
71
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_storage_info.test
vendored
Normal file
71
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_storage_info.test
vendored
Normal file
@@ -0,0 +1,71 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_storage_info.test
|
||||
# description: Test storage info with Bitpacking
|
||||
# group: [bitpacking]
|
||||
|
||||
# This test defaults to another compression function for smaller block sizes,
|
||||
# because the bitpacking groups no longer fit the blocks.
|
||||
require block_size 262144
|
||||
|
||||
require vector_size 2048
|
||||
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression = 'bitpacking'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test (a INTEGER, b INTEGER);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test VALUES (11, 22), (11, 22), (12, 21), (NULL, NULL)
|
||||
|
||||
statement ok
|
||||
CHECKPOINT
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'INTEGER' LIMIT 1
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_bp (a INTEGER)
|
||||
|
||||
statement ok
|
||||
INSERT INTO test_bp SELECT 1 FROM range(0, 10000) tbl(i)
|
||||
|
||||
statement ok
|
||||
INSERT INTO test_bp SELECT 2 FROM range(0, 10000) tbl(i)
|
||||
|
||||
statement ok
|
||||
CHECKPOINT
|
||||
|
||||
query I
|
||||
SELECT
|
||||
segment_info
|
||||
FROM
|
||||
pragma_storage_info('test_bp')
|
||||
WHERE segment_type NOT IN ('VALIDITY')
|
||||
----
|
||||
CONSTANT: 9, DELTA_FOR: 1
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode = 'delta_for'
|
||||
|
||||
statement ok
|
||||
CREATE OR REPLACE TABLE test_bp (a INTEGER)
|
||||
|
||||
statement ok
|
||||
INSERT INTO test_bp SELECT 3*(i // 1000) + (i%10) FROM range(0, 10000) tbl(i)
|
||||
|
||||
statement ok
|
||||
CHECKPOINT
|
||||
|
||||
# Should have all 5 blocks for 10K integers (1 block per 2048 tuples) forced to DELTA_FOR
|
||||
query I
|
||||
SELECT
|
||||
segment_info
|
||||
FROM
|
||||
pragma_storage_info('test_bp')
|
||||
WHERE segment_type NOT IN ('VALIDITY')
|
||||
----
|
||||
DELTA_FOR: 5
|
||||
49
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_table_copy.test
vendored
Normal file
49
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_table_copy.test
vendored
Normal file
@@ -0,0 +1,49 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_table_copy.test
|
||||
# description: Tests a table copy on a table spanning multiple segments
|
||||
# group: [bitpacking]
|
||||
|
||||
require tpch
|
||||
|
||||
# load the DB from disk
|
||||
load __TEST_DIR__/test_bitpacking_struct_bug.db
|
||||
|
||||
foreach bitpacking_mode delta_for for constant_delta
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='${bitpacking_mode}'
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression = 'bitpacking'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test (a integer);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i FROM range(0,150000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_2 AS SELECT a FROM test;
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select sum(a) from test;
|
||||
----
|
||||
11249925000
|
||||
|
||||
query I
|
||||
select sum(a) from test_2;
|
||||
----
|
||||
11249925000
|
||||
|
||||
statement ok
|
||||
drop table test
|
||||
|
||||
statement ok
|
||||
drop table test_2
|
||||
|
||||
endloop
|
||||
126
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_types.test_slow
vendored
Normal file
126
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_types.test_slow
vendored
Normal file
@@ -0,0 +1,126 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_types.test_slow
|
||||
# description: Test bitpacking with different types, especially around the numerical limits
|
||||
# group: [bitpacking]
|
||||
|
||||
# This test defaults to another compression function for smaller block sizes,
|
||||
# because the bitpacking groups no longer fit the blocks.
|
||||
require block_size 262144
|
||||
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression = 'bitpacking'
|
||||
|
||||
foreach bitpacking_mode delta_for for constant_delta constant
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='${bitpacking_mode}'
|
||||
|
||||
foreach type <numeric> decimal(4,1) decimal(8,1) decimal(12,1) decimal(18,1)
|
||||
|
||||
statement ok
|
||||
CREATE TABLE a AS SELECT MOD(i,3)::${type} i FROM range(10000) tbl(i)
|
||||
|
||||
query IIIII
|
||||
SELECT MIN(i), MAX(i), AVG(i), COUNT(*), COUNT(i) FROM a
|
||||
----
|
||||
0 2 0.999900 10000 10000
|
||||
|
||||
query IIIII
|
||||
SELECT MIN(i), MAX(i), AVG(i), COUNT(*), COUNT(i) FROM a WHERE i=1
|
||||
----
|
||||
1 1 1.000000 3333 3333
|
||||
|
||||
statement ok
|
||||
DROP TABLE a
|
||||
|
||||
endloop
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test (d INT32);
|
||||
|
||||
# Range too big to force bitpacking
|
||||
statement ok
|
||||
INSERT INTO test VALUES (-2147483648), (2147483647);
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE (
|
||||
segment_type ILIKE 'INTEGER')
|
||||
----
|
||||
Uncompressed
|
||||
|
||||
statement ok
|
||||
DROP TABLE IF EXISTS test;
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test (x INT128, a INT64, b INT32, c INT16, d TINYINT);
|
||||
|
||||
# Only using (SIZEOF(TYPE)*8)-1
|
||||
# (All but 1 bit)
|
||||
# Because bitpacking wont accept a value range bigger than that
|
||||
statement ok
|
||||
INSERT INTO test VALUES (-85070591730234615865843651857942052864, -4611686018427387904, -1073741824, -16384, -64), (85070591730234615865843651857942052863, 4611686018427387903, 1073741823, 16383, 63);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE (
|
||||
segment_type ILIKE 'HUGEINT' OR
|
||||
segment_type ILIKE 'BIGINT' OR
|
||||
segment_type ILIKE 'INTEGER' OR
|
||||
segment_type ILIKE 'SMALLINT' OR
|
||||
segment_type ILIKE 'TINYINT')
|
||||
----
|
||||
BitPacking
|
||||
BitPacking
|
||||
BitPacking
|
||||
BitPacking
|
||||
BitPacking
|
||||
|
||||
query IIIIII
|
||||
SELECT AVG(x), AVG(a), AVG(b), AVG(c), AVG(d), COUNT(*) FROM test
|
||||
----
|
||||
-0.5 -0.5 -0.5 -0.5 -0.5 2
|
||||
|
||||
statement ok
|
||||
drop table test
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test (a UINT64, b UINT32, c UINT16, d UINT8);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i, i, i, i FROM range(0, 256) tbl(i);
|
||||
INSERT INTO test SELECT i, i, i, NULL FROM range(31768, 32768) tbl(i);
|
||||
INSERT INTO test SELECT i, i, NULL, NULL FROM range(4294966295, 4294967295) tbl(i);
|
||||
INSERT INTO test SELECT 18446744073709551615 - i, NULL, NULL, NULL FROM range(0, 1000) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query IIIII
|
||||
SELECT AVG(a), AVG(b), AVG(c), AVG(d), COUNT(*) FROM test
|
||||
----
|
||||
5.665461940419088e+18 1903811655.4255319 25716.671974522294 127.5 3256
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_bool (id VARCHAR, col BOOL)
|
||||
|
||||
statement ok
|
||||
INSERT INTO test_bool SELECT i::VARCHAR id, CAST (i%2 as BOOL) col FROM range(10000) tbl(i)
|
||||
|
||||
statement ok
|
||||
CHECKPOINT
|
||||
|
||||
query I
|
||||
SELECT COUNT(*) FROM test_bool WHERE col = TRUE
|
||||
----
|
||||
5000
|
||||
|
||||
statement ok
|
||||
drop table test
|
||||
|
||||
statement ok
|
||||
drop table test_bool
|
||||
|
||||
endloop
|
||||
255
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_uhugeint.test
vendored
Normal file
255
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_uhugeint.test
vendored
Normal file
@@ -0,0 +1,255 @@
|
||||
# name: test/sql/storage/compression/bitpacking/bitpacking_uhugeint.test
|
||||
# description: Test uhugeint bitpacking at multiple bitwidths
|
||||
# group: [bitpacking]
|
||||
|
||||
# This test defaults to another compression function for smaller block sizes,
|
||||
# because the bitpacking groups no longer fit the blocks.
|
||||
require block_size 262144
|
||||
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression='bitpacking'
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='constant'
|
||||
|
||||
# bit-width < 32
|
||||
# ----------------------------------------
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a UHUGEINT);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, i::UHUGEINT FROM range(0, 16) tbl(i);
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0, 16) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
0
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
|
||||
# make sure compression is being used
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'UHUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
# bit-width == 32
|
||||
# ----------------------------------------
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a UHUGEINT);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, i::UHUGEINT + 3000000000::UHUGEINT FROM range(0, 16) tbl(i);
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0, 16) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
3000000000
|
||||
3000000001
|
||||
3000000002
|
||||
3000000003
|
||||
3000000004
|
||||
|
||||
# make sure compression is being used
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'UHUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
# 32 < bit-width < 64
|
||||
# ----------------------------------------
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a UHUGEINT);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, i::UHUGEINT + 200000000000::UHUGEINT FROM range(0, 16) tbl(i);
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0, 16) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
200000000000
|
||||
200000000001
|
||||
200000000002
|
||||
200000000003
|
||||
200000000004
|
||||
|
||||
# make sure compression is being used
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'UHUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
# bit-width == 64
|
||||
# ----------------------------------------
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a UHUGEINT);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, i::UHUGEINT + 10000000000000000000::UHUGEINT FROM range(0, 16) tbl(i);
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0, 16) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
10000000000000000000
|
||||
10000000000000000001
|
||||
10000000000000000002
|
||||
10000000000000000003
|
||||
10000000000000000004
|
||||
|
||||
# make sure compression is being used
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'UHUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
# 64 < bit-width < 96
|
||||
# ----------------------------------------
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a UHUGEINT);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, i::UHUGEINT + 500000000000000000000::UHUGEINT FROM range(0, 16) tbl(i);
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0, 16) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
500000000000000000000
|
||||
500000000000000000001
|
||||
500000000000000000002
|
||||
500000000000000000003
|
||||
500000000000000000004
|
||||
|
||||
# make sure compression is being used
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'UHUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
# bit-width == 96
|
||||
# ----------------------------------------
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a UHUGEINT);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, i::UHUGEINT + 50000000000000000000000000000::UHUGEINT FROM range(0, 16) tbl(i);
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0, 16) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
50000000000000000000000000000
|
||||
50000000000000000000000000001
|
||||
50000000000000000000000000002
|
||||
50000000000000000000000000003
|
||||
50000000000000000000000000004
|
||||
|
||||
# make sure compression is being used
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'UHUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
# 96 < bit-width < 128
|
||||
# ----------------------------------------
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a UHUGEINT);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, i::UHUGEINT + 300000000000000000000000000000000::UHUGEINT FROM range(0, 16) tbl(i);
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0, 16) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
300000000000000000000000000000000
|
||||
300000000000000000000000000000001
|
||||
300000000000000000000000000000002
|
||||
300000000000000000000000000000003
|
||||
300000000000000000000000000000004
|
||||
|
||||
# make sure compression is being used
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'UHUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
# bit-width == 128
|
||||
# ----------------------------------------
|
||||
statement ok
|
||||
CREATE TABLE test (id VARCHAR, a UHUGEINT);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT i::VARCHAR, i::UHUGEINT + 20000000000000000000000000000000000000::UHUGEINT FROM range(0, 16) tbl(i);
|
||||
INSERT INTO test SELECT i::VARCHAR, 13371337 FROM range(0, 16) tbl(i);
|
||||
|
||||
statement ok
|
||||
checkpoint
|
||||
|
||||
query I
|
||||
select a from test limit 5;
|
||||
----
|
||||
20000000000000000000000000000000000000
|
||||
20000000000000000000000000000000000001
|
||||
20000000000000000000000000000000000002
|
||||
20000000000000000000000000000000000003
|
||||
20000000000000000000000000000000000004
|
||||
|
||||
# make sure compression is being used
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'UHUGEINT';
|
||||
----
|
||||
BitPacking
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
31
external/duckdb/test/sql/storage/compression/bitpacking/force_bitpacking.test
vendored
Normal file
31
external/duckdb/test/sql/storage/compression/bitpacking/force_bitpacking.test
vendored
Normal file
@@ -0,0 +1,31 @@
|
||||
# name: test/sql/storage/compression/bitpacking/force_bitpacking.test
|
||||
# description: Test forcing bitpacking as the compression scheme
|
||||
# group: [bitpacking]
|
||||
|
||||
# This test defaults to another compression function for smaller block sizes,
|
||||
# because the bitpacking groups no longer fit the blocks.
|
||||
require block_size 262144
|
||||
|
||||
require vector_size 2048
|
||||
|
||||
load __TEST_DIR__/force_bitpacking.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression = 'bitpacking'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test_bp (a INTEGER);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test_bp SELECT 1 FROM range(0, 1000) tbl(i);
|
||||
|
||||
statement ok
|
||||
INSERT INTO test_bp SELECT 2 FROM range(0, 1000) tbl(i);
|
||||
|
||||
statement ok
|
||||
CHECKPOINT
|
||||
|
||||
query I
|
||||
SELECT compression FROM pragma_storage_info('test_bp') WHERE segment_type ILIKE 'INTEGER' LIMIT 1
|
||||
----
|
||||
BitPacking
|
||||
40
external/duckdb/test/sql/storage/compression/bitpacking/struct_bitpacking.test
vendored
Normal file
40
external/duckdb/test/sql/storage/compression/bitpacking/struct_bitpacking.test
vendored
Normal file
@@ -0,0 +1,40 @@
|
||||
# name: test/sql/storage/compression/bitpacking/struct_bitpacking.test
|
||||
# description: Test storage with Bitpacking inside structs
|
||||
# group: [bitpacking]
|
||||
|
||||
# load the DB from disk
|
||||
load __TEST_DIR__/test_bitpacking.db
|
||||
|
||||
statement ok
|
||||
PRAGMA force_compression = 'bitpacking'
|
||||
|
||||
foreach bitpacking_mode delta_for for constant_delta constant
|
||||
|
||||
statement ok
|
||||
PRAGMA force_bitpacking_mode='${bitpacking_mode}'
|
||||
|
||||
statement ok
|
||||
CREATE TABLE test (s ROW(a INTEGER));
|
||||
|
||||
statement ok
|
||||
INSERT INTO test SELECT {'a': i} FROM range(0, 10000) tbl(i);
|
||||
INSERT INTO test SELECT {'a': i} FROM range(22767, 32767) tbl(i);
|
||||
INSERT INTO test SELECT {'a': 1337} FROM range(2147473647, 2147483647) tbl(i);
|
||||
INSERT INTO test SELECT {'a': i} FROM range(2147473647, 2147483647) tbl(i);
|
||||
|
||||
query IIII
|
||||
SELECT SUM(s['a']), MIN(s['a']), MAX(s['a']), COUNT(*) FROM test
|
||||
----
|
||||
21475127495000 0 2147483646 40000
|
||||
|
||||
restart
|
||||
|
||||
query IIII
|
||||
SELECT SUM(s['a']), MIN(s['a']), MAX(s['a']), COUNT(*) FROM test
|
||||
----
|
||||
21475127495000 0 2147483646 40000
|
||||
|
||||
statement ok
|
||||
DROP TABLE test;
|
||||
|
||||
endloop
|
||||
Reference in New Issue
Block a user