marsultor/email-tracker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

should be it

Browse Source
This commit is contained in:
Mars Ultor
2025-10-24 19:21:19 -05:00
parent a4b23fc57c
commit f09560c7b1
14047 changed files with 3161551 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
external/duckdb/test/sql/storage/CMakeLists.txt vendored Normal file
View File

@@ -0,0 +1,12 @@
add_subdirectory(catalog)
add_library_unity(
test_sql_storage
OBJECT
test_buffer_manager.cpp
test_checksum.cpp
test_storage.cpp
test_database_size.cpp
wal_torn_write.cpp)
set(ALL_OBJECT_FILES
${ALL_OBJECT_FILES} $<TARGET_OBJECTS:test_sql_storage>
PARENT_SCOPE)

23
external/duckdb/test/sql/storage/all_types_storage.test_slow vendored Normal file
View File

@@ -0,0 +1,23 @@
# name: test/sql/storage/all_types_storage.test_slow
# description: Test all types function
# group: [storage]
# load the DB from disk
load __TEST_DIR__/all_types_storage.db
statement ok
PRAGMA enable_verification
statement ok
CREATE TABLE all_types AS SELECT * FROM test_all_types();
query IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII nosort r1
SELECT * FROM test_all_types();
query IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII nosort r1
SELECT * FROM all_types;
restart
query IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII nosort r1
SELECT * FROM all_types;

37
external/duckdb/test/sql/storage/all_types_storage_large.test_slow vendored Normal file
View File

@@ -0,0 +1,37 @@
# name: test/sql/storage/all_types_storage_large.test_slow
# description: Test all types storage (large)
# group: [storage]
# load the DB from disk
load __TEST_DIR__/all_types_storage.db
statement ok
PRAGMA enable_verification
foreach compression <compression>
statement ok
PRAGMA force_compression='${compression}'
statement ok
CREATE TABLE all_types AS SELECT * FROM test_all_types();
loop i 0 12
statement ok
INSERT INTO all_types SELECT * FROM all_types;
endloop
query IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII nosort r1
SELECT * FROM all_types;
restart
query IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII nosort r1
SELECT * FROM all_types;
statement ok
DROP TABLE all_types;
endloop

85
external/duckdb/test/sql/storage/alter/alter_column_constraint.test vendored Normal file
View File

@@ -0,0 +1,85 @@
# name: test/sql/storage/alter/alter_column_constraint.test
# description: Issue #1785: ALTER TABLE causes constraints lost in database file after exiting
# group: [alter]
# load the DB from disk
load __TEST_DIR__/add_column_constraint.db
statement ok
CREATE TABLE IF NOT EXISTS a(id INT PRIMARY KEY);
statement ok
INSERT INTO a(id) VALUES (1);
# add column
statement ok
ALTER TABLE a ADD COLUMN c REAL;
statement error
INSERT INTO a(id) VALUES (1);
----
restart
statement error
INSERT INTO a(id) VALUES (1);
----
# add default
statement ok
ALTER TABLE a ALTER COLUMN c SET DEFAULT 10;
statement error
INSERT INTO a(id) VALUES (1);
----
restart
statement error
INSERT INTO a(id) VALUES (1);
----
# rename column
statement ok
ALTER TABLE a RENAME c TO d;
statement error
INSERT INTO a(id) VALUES (1);
----
restart
statement error
INSERT INTO a(id) VALUES (1);
----
# rename table
statement ok
ALTER TABLE a RENAME TO b;
statement error
INSERT INTO b(id) VALUES (1);
----
restart
statement error
INSERT INTO b(id) VALUES (1);
----
# drop column
statement ok
ALTER TABLE b DROP d;
statement error
INSERT INTO b(id) VALUES (1);
----
restart
statement error
INSERT INTO b(id) VALUES (1);
----
statement ok
INSERT INTO b(id) VALUES (2);

35
external/duckdb/test/sql/storage/append_strings_to_persistent.test vendored Normal file
View File

@@ -0,0 +1,35 @@
# name: test/sql/storage/append_strings_to_persistent.test
# description: Append strings to a block that has been checkpointed
# group: [storage]
# load the DB from disk
load __TEST_DIR__/append_strings_to_persistent.db
statement ok
CREATE TABLE vals(i INTEGER, v VARCHAR)
statement ok
INSERT INTO vals VALUES (1, 'hello')
statement ok
INSERT INTO vals SELECT i, i::VARCHAR FROM generate_series(2,10000) t(i)
query IIII
SELECT MIN(i), MAX(i), MIN(v), MAX(v) FROM vals
----
1 10000 10 hello
restart
query IIII
SELECT MIN(i), MAX(i), MIN(v), MAX(v) FROM vals
----
1 10000 10 hello
statement ok
INSERT INTO vals SELECT i, i::VARCHAR FROM generate_series(10001,100000) t(i)
query IIII
SELECT MIN(i), MAX(i), MIN(v), MAX(v) FROM vals
----
1 100000 10 hello

18
external/duckdb/test/sql/storage/bc/internal_schemas_0102.test vendored Normal file
View File

@@ -0,0 +1,18 @@
# name: test/sql/storage/bc/internal_schemas_0102.test
# description: Verify internal schemas are not duplicated in the main schema when reading older databases
# group: [bc]
# The database is written with a vector size of 2048.
require vector_size 2048
unzip data/storage/bc_0102.db.gz __TEST_DIR__/bc_0102.db
load __TEST_DIR__/bc_0102.db readonly
statement ok
PRAGMA enable_verification
query II
SELECT database_name, schema_name FROM duckdb_schemas WHERE NOT internal
----
bc_0102 s1

34
external/duckdb/test/sql/storage/bc/test_view_v092.test vendored Normal file
View File

@@ -0,0 +1,34 @@
# name: test/sql/storage/bc/test_view_v092.test
# description: Test reading database with views
# group: [bc]
# The database is written with a vector size of 2048.
require vector_size 2048
load data/storage/views_092.db readonly
statement ok
PRAGMA enable_verification
# query the views directly
query I
SELECT * FROM v1
----
query I
SELECT * FROM v2
----
query I
SELECT * FROM v3
----
# run metadata functions
statement ok
SHOW TABLES
statement ok
FROM duckdb_views()
statement ok
FROM duckdb_columns()

78
external/duckdb/test/sql/storage/buffer_manager.test_slow vendored Normal file
View File

@@ -0,0 +1,78 @@
# name: test/sql/storage/buffer_manager.test_slow
# description: Test that the buffer manager correctly frees any memory we use
# group: [storage]
require skip_reload
load
statement ok
PRAGMA temp_directory=''
statement ok
PRAGMA threads=1
statement ok
PRAGMA memory_limit='2MB'
# streaming results do not use any memory
query II
SELECT MIN(i), MAX(i) FROM range(1000001) tbl(i)
----
0 1000000
# increase memory limit 10MB because we always partition data in GROUP BY queries now (for potential out-of-core)
statement ok
PRAGMA memory_limit='10MB'
# we can group by 9K integers (fits into one block of 256KB)
statement ok
SELECT i, MIN(i), MAX(i) FROM range(9000) tbl(i) GROUP BY i
# 1M integers is too much -> does not fit into 1MB
statement error
SELECT i, MIN(i), MAX(i) FROM range(1000000) tbl(i) GROUP BY i
----
# we can group by 9K integers a bunch of times -> memory should be released between queries
loop i 0 10
statement ok
SELECT i, MIN(i), MAX(i) FROM range(9000) tbl(i) GROUP BY i
endloop
# set the memory limit back to 2MB again, no more GROUP BY's in this test
statement ok
PRAGMA memory_limit='2MB'
# we can create a single table with 10K integers
statement ok
CREATE TABLE t1 AS SELECT * FROM range(10000) tbl(i)
# we can't create a table with 1M integers: not enough memory!
statement error
CREATE TABLE t2 AS SELECT * FROM range(1000000) tbl(i)
----
# if we drop the table, we can re-use any memory that the table took up (i.e. the memory gets released again)
loop i 0 10
statement ok
DROP TABLE t1;
statement ok
CREATE TABLE t1 AS SELECT * FROM range(10000) tbl(i)
endloop
# same with strings
loop i 0 10
statement ok
DROP TABLE t1;
statement ok
CREATE TABLE t1 AS SELECT i::VARCHAR FROM range(10000) tbl(i)
endloop

53
external/duckdb/test/sql/storage/buffer_manager/appending_table_exceeding_limit.test_slow vendored Normal file
View File

@@ -0,0 +1,53 @@
# name: test/sql/storage/buffer_manager/appending_table_exceeding_limit.test_slow
# description: Test appending and checkpointing a table that exceeds buffer manager size
# group: [buffer_manager]
# load the DB from disk
load __TEST_DIR__/test_table_exceeding_limit.db
statement ok
SET force_compression='uncompressed'
statement ok
SET memory_limit = '10MB'
statement ok
SET threads=1
statement ok
CREATE TABLE test (a INTEGER, b INTEGER);
statement ok
INSERT INTO test VALUES (1, 10), (2, 20), (3, 30), (NULL, NULL)
loop i 0 23
statement ok
INSERT INTO test SELECT * FROM test
endloop
query IIII
SELECT COUNT(*), COUNT(a), SUM(a), SUM(b) FROM test
----
33554432 25165824 50331648 503316480
loop i 0 2
restart
statement ok
SET force_compression='uncompressed'
statement ok
SET memory_limit = '10MB'
statement ok
SET threads=1
query IIII
SELECT COUNT(*), COUNT(a), SUM(a), SUM(b) FROM test
----
33554432 25165824 50331648 503316480
endloop

56
external/duckdb/test/sql/storage/buffer_manager/larger_than_memory_aggregate.test_slow vendored Normal file
View File

@@ -0,0 +1,56 @@
# name: test/sql/storage/buffer_manager/larger_than_memory_aggregate.test_slow
# description: Test scanning a table and computing an aggregate over a table that exceeds buffer manager size
# group: [buffer_manager]
# load the DB from disk
load __TEST_DIR__/larger_than_memory_aggregate.db
statement ok
PRAGMA force_compression='uncompressed'
statement ok
SET memory_limit='10000000b'
statement ok
SET threads=1
statement ok
CREATE TABLE test (a INTEGER, b INTEGER);
statement ok
INSERT INTO test VALUES (11, 22), (13, 22), (12, 21), (NULL, NULL)
# insert until 16777216 rows
loop i 0 22
statement ok
INSERT INTO test SELECT * FROM test
endloop
query I
SELECT COUNT(*) FROM test
----
16777216
query I
SELECT SUM(a) + SUM(b) FROM test
----
423624704
loop i 0 2
restart
statement ok
SET memory_limit='10000000b'
statement ok
SET threads=1
query I
SELECT SUM(a) + SUM(b) FROM test
----
423624704
endloop

32
external/duckdb/test/sql/storage/buffer_manager_temp_dir.test vendored Normal file
View File

@@ -0,0 +1,32 @@
# name: test/sql/storage/buffer_manager_temp_dir.test
# description: Test setting of the temporary directory
# group: [storage]
require noforcestorage
# Set the temporary directory to an empty directory.
statement ok
PRAGMA temp_directory=''
statement ok
PRAGMA memory_limit='2MB'
# we can't offload anything to disk now
# hence this creation should fail as the table does not fit within our memory limit
statement error
CREATE TABLE t2 AS SELECT * FROM range(1000000);
----
<REGEX>:.*Out of Memory Error.*could not allocate block of size.*
# after we set the temp directory, we can create this table
statement ok
PRAGMA temp_directory='__TEST_DIR__/myfile.tmp'
statement ok
CREATE TABLE t2 AS SELECT * FROM range(1000000);
# after the temp directory is created, we can't change the temp directory again
statement error
PRAGMA temp_directory=''
----
<REGEX>:.*Not implemented Error.*Cannot switch temporary directory.*

5
external/duckdb/test/sql/storage/catalog/CMakeLists.txt vendored Normal file
View File

@@ -0,0 +1,5 @@
add_library_unity(test_sql_storage_catalog OBJECT test_storage_sequences.cpp)
set(ALL_OBJECT_FILES
${ALL_OBJECT_FILES} $<TARGET_OBJECTS:test_sql_storage_catalog>
PARENT_SCOPE)

32
external/duckdb/test/sql/storage/catalog/generated_columns/virtual/basic.test vendored Normal file
View File

@@ -0,0 +1,32 @@
# name: test/sql/storage/catalog/generated_columns/virtual/basic.test
# description: Create a table with basic generated column usage and restart
# group: [virtual]
statement ok
PRAGMA enable_verification
# load the DB from disk
load __TEST_DIR__/generated_column_storage.db
statement ok
CREATE TABLE tbl (
price INTEGER,
gcol AS (price)
);
restart
statement ok
INSERT INTO tbl VALUES (5);
query I
SELECT gcol FROM tbl;
----
5
restart
query I
SELECT gcol FROM tbl;
----
5

203
external/duckdb/test/sql/storage/catalog/generated_columns/virtual/constraints.test vendored Normal file
View File

@@ -0,0 +1,203 @@
# name: test/sql/storage/catalog/generated_columns/virtual/constraints.test
# description: Constraints combined with generated columns
# group: [virtual]
statement ok
PRAGMA enable_verification
# load the DB from disk
load __TEST_DIR__/generated_column_storage.db
# --- CHECK - indirect
statement ok
CREATE TABLE tbl (
gcol_x AS (x),
gcol_y AS (y),
y INTEGER,
x TEXT
CHECK (y > 5)
);
restart
statement error
INSERT INTO tbl VALUES (5, 'test');
----
restart
statement ok
INSERT INTO tbl VALUES (6,'test');
restart
statement ok
INSERT INTO tbl VALUES (6,'test');
restart
statement ok
SELECT * FROM tbl;
statement ok
DROP TABLE tbl;
# --- CHECK - direct
statement ok
CREATE TABLE tbl (
gcol_x AS (x),
gcol_y AS (y),
y INTEGER CHECK (y > 5),
x TEXT
);
restart
statement error
INSERT INTO tbl VALUES (5, 'test');
----
restart
statement ok
INSERT INTO tbl VALUES (6,'test');
restart
statement ok
INSERT INTO tbl VALUES (6,'test');
restart
statement ok
SELECT * FROM tbl;
statement ok
DROP TABLE tbl;
# --- UNIQUE | PRIMARY direct
statement ok
CREATE TABLE tbl (
gcol_x AS (x),
gcol_y AS (y),
y INTEGER UNIQUE,
x TEXT PRIMARY KEY
);
restart
statement ok
INSERT INTO tbl VALUES (6,'test');
restart
statement error
INSERT INTO tbl VALUES (6,'test');
----
restart
statement ok
SELECT * FROM tbl;
statement ok
DROP TABLE tbl;
# --- UNIQUE | PRIMARY indirect
statement ok
CREATE TABLE tbl (
gcol_x AS (x),
gcol_y AS (y),
y INTEGER,
x TEXT,
PRIMARY KEY (y),
UNIQUE (x)
);
restart
statement ok
INSERT INTO tbl VALUES (6,'test');
restart
statement error
INSERT INTO tbl VALUES (6,'test');
----
restart
statement ok
SELECT * FROM tbl;
statement ok
DROP TABLE tbl;
# --- FOREIGN KEY
statement ok
CREATE TABLE base (
price INTEGER PRIMARY KEY
);
statement ok
CREATE TABLE tbl (
gcol_nest AS (gcol),
gcol AS (x),
x INTEGER,
FOREIGN KEY (x) REFERENCES base (price)
);
restart
statement ok
INSERT INTO base VALUES (5);
statement ok
DROP TABLE tbl;
statement ok
DROP TABLE base;
# --- DEFAULT value
statement ok
CREATE TABLE tbl (
gcol2 AS (gcol1),
price INTEGER DEFAULT (5),
gcol1 AS (price),
);
restart
statement ok
INSERT INTO tbl VALUES (DEFAULT);
restart
query III
SELECT * FROM tbl;
----
5 5 5
# gcol1 is a dependency of gcol2
statement error
ALTER TABLE tbl DROP COLUMN gcol1;
----
# gcol2 is not a dependency of anything
statement ok
ALTER TABLE tbl DROP COLUMN gcol2;
# gcol1 is no longer a dependency of anything
statement ok
ALTER TABLE tbl DROP COLUMN gcol1;
restart
statement ok
SELECT * FROM tbl;

42
external/duckdb/test/sql/storage/catalog/store_collate.test vendored Normal file
View File

@@ -0,0 +1,42 @@
# name: test/sql/storage/catalog/store_collate.test
# description: Test storage of collations
# group: [catalog]
# load the DB from disk
load __TEST_DIR__/collate_test.db
statement ok
CREATE TABLE collate_test(s VARCHAR COLLATE NOACCENT)
statement ok
INSERT INTO collate_test VALUES ('Mühleisen'), ('Hëllö')
# collate in equality
query T
SELECT * FROM collate_test WHERE s='Muhleisen'
----
Mühleisen
statement ok
SELECT * FROM collate_test WHERE s='mühleisen'
query T
SELECT * FROM collate_test WHERE s='Hello'
----
Hëllö
restart
# collate in equality
query T
SELECT * FROM collate_test WHERE s='Muhleisen'
----
Mühleisen
statement ok
SELECT * FROM collate_test WHERE s='mühleisen'
query T
SELECT * FROM collate_test WHERE s='Hello'
----
Hëllö

39
external/duckdb/test/sql/storage/catalog/test_check_constraint.test vendored Normal file
View File

@@ -0,0 +1,39 @@
# name: test/sql/storage/catalog/test_check_constraint.test
# description: Test serialization of CHECK constraint
# group: [catalog]
# load the DB from disk
load __TEST_DIR__/check_storage_test.db
# create a table with a check constraint
statement ok
CREATE TABLE test(a INTEGER CHECK (a<10), b INTEGER CHECK(CASE WHEN b < 10 THEN a < b ELSE a + b < 100 END));
# reload the database
restart
# matching tuple
statement ok
INSERT INTO test VALUES (3, 7);
# check constraint on a violated (a < 10)
statement error
INSERT INTO test VALUES (12, 13);
----
<REGEX>:.*Constraint Error.*CHECK constraint failed on table.*
# check constraint on b violated (b < 10) => (a < b)
statement error
INSERT INTO test VALUES (5, 3);
----
<REGEX>:.*Constraint Error.*CHECK constraint failed on table.*
# check constraint on b not violated !(b < 10) => (a + b < 100)
statement ok
INSERT INTO test VALUES (9, 90);
# check constraint on b violated !(b < 10) => (a + b < 100)
statement error
INSERT INTO test VALUES (9, 99);
----
<REGEX>:.*Constraint Error.*CHECK constraint failed on table.*

53
external/duckdb/test/sql/storage/catalog/test_drop_table.test vendored Normal file
View File

@@ -0,0 +1,53 @@
# name: test/sql/storage/catalog/test_drop_table.test
# description: Test serialization of NOT NULL constraint
# group: [catalog]
# load the DB from disk
load __TEST_DIR__/test_drop_table.db
# create a schema and table and insert values
statement ok
CREATE SCHEMA test;
statement ok
CREATE TABLE test.test (a INTEGER, b INTEGER);
statement ok
INSERT INTO test.test VALUES (11, 22), (13, 22);
# drop the schema/table
statement ok
DROP TABLE test.test
statement ok
DROP SCHEMA test
# now create then again
statement ok
CREATE SCHEMA test;
statement ok
CREATE TABLE test.test (a INTEGER, b INTEGER);
statement ok
INSERT INTO test.test VALUES (11, 22), (13, 22);
# restart the database
restart
# after reload the table and schema are there: we can drop them again
statement ok
DROP TABLE test.test
statement ok
DROP SCHEMA test
restart
# after another reload they are still gone: create them again
statement ok
CREATE SCHEMA test;
statement ok
CREATE TABLE test.test (a INTEGER, b INTEGER);

87
external/duckdb/test/sql/storage/catalog/test_macro_storage.test vendored Normal file
View File

@@ -0,0 +1,87 @@
# name: test/sql/storage/catalog/test_macro_storage.test
# description: Create and drop a macro over different runs
# group: [catalog]
# load the DB from disk
load __TEST_DIR__/macro_storage.db
statement ok
PRAGMA disable_checkpoint_on_shutdown
statement ok
PRAGMA wal_autocheckpoint='1TB';
# create a macro
statement ok
CREATE MACRO plus1(a) AS a+1
# use the macro
query T
SELECT plus1(2)
----
3
statement ok
DROP MACRO plus1
loop i 0 2
# restart the system
restart
statement error
SELECT plus1(2)
----
# after recreating the macro we can use it again
statement ok
CREATE MACRO plus1(a) AS a+1
query T
SELECT plus1(2)
----
3
# drop the macro again
statement ok
DROP MACRO plus1
endloop
# create a macro without deleting it this time
statement ok
CREATE MACRO plus2(a, b := 2) AS a + b
loop i 0 2
query T
SELECT plus2(3)
----
5
restart
query T
SELECT plus2(4)
----
6
endloop
# macro overloads
# create a macro
statement ok
CREATE MACRO addition(a) AS a, (a,b) AS a + b
# use the macro
query II
SELECT addition(2), addition(1, 2)
----
2 3
restart
query II
SELECT addition(2), addition(1, 2)
----
2 3

35
external/duckdb/test/sql/storage/catalog/test_not_distinct_from_default.test vendored Normal file
View File

@@ -0,0 +1,35 @@
# name: test/sql/storage/catalog/test_not_distinct_from_default.test
# description: Test using NOT DISTINCT FROM in a default expression
# group: [catalog]
# Use in DEFAULT expression of a table
load __TEST_DIR__/not_distinct_test.db
statement ok
CREATE SEQUENCE seq;
statement ok
CREATE TABLE test_default (a BOOL DEFAULT nextval('seq') is not distinct from nextval('seq'), b INTEGER);
statement ok
INSERT INTO test_default (b) VALUES (2), (4), (6);
query II
select * from test_default;
----
False 2
False 4
False 6
# Use in DEFAULT expression of a table and store that table on disk (storage test)
restart
query II
select * from test_default;
----
False 2
False 4
False 6

29
external/duckdb/test/sql/storage/catalog/test_not_null_constraint.test vendored Normal file
View File

@@ -0,0 +1,29 @@
# name: test/sql/storage/catalog/test_not_null_constraint.test
# description: Test serialization of NOT NULL constraint
# group: [catalog]
# load the DB from disk
load __TEST_DIR__/not_null_storage_test.db
statement ok
PRAGMA disable_checkpoint_on_shutdown
# create a table with a check constraint
statement ok
CREATE TABLE test(a INTEGER NOT NULL);
loop i 0 2
# reload the database
restart
# constraint failure
statement error
INSERT INTO test VALUES (NULL)
----
# constraint ok
statement ok
INSERT INTO test VALUES (1)
endloop

91
external/duckdb/test/sql/storage/catalog/test_prepared_storage.test vendored Normal file
View File

@@ -0,0 +1,91 @@
# name: test/sql/storage/catalog/test_prepared_storage.test
# description: PREPARE and WAL
# group: [catalog]
# load the DB from disk
load __TEST_DIR__/prepared_storage.db
# insert values in a database using prepared statements
statement ok
CREATE TABLE t (a INTEGER)
statement ok
PREPARE p1 AS INSERT INTO t VALUES ($1)
statement ok
EXECUTE p1(42)
statement ok
EXECUTE p1(43)
statement ok
DEALLOCATE p1
query I
SELECT a FROM t
----
42
43
restart
# now restart and verify that the data is still there
query I
SELECT a FROM t
----
42
43
# unhelpfully use the same statement name again, it should be available, but do nothing with it
statement ok
PREPARE p1 AS DELETE FROM t WHERE a=$1
restart
# do the same with delete
statement ok
PREPARE p1 AS DELETE FROM t WHERE a=$1
statement ok
EXECUTE p1(43)
query I
SELECT a FROM t
----
42
# restart again
restart
# data is still gone
query I
SELECT a FROM t
----
42
# now with update
restart
query I
SELECT a FROM t
----
42
statement ok
PREPARE p1 AS UPDATE t SET a = $1
statement ok
EXECUTE p1(43)
query I
SELECT a FROM t
----
43
restart
query I
SELECT a FROM t
----
43

33
external/duckdb/test/sql/storage/catalog/test_sequence_uncommitted_transaction.test vendored Normal file
View File

@@ -0,0 +1,33 @@
# name: test/sql/storage/catalog/test_sequence_uncommitted_transaction.test
# description: Use sequences with uncommited transaction
# group: [catalog]
load __TEST_DIR__/store_sequences.db
statement ok
CREATE SEQUENCE seq
# start a transaction in con2
statement ok con2
BEGIN transaction
# select the nextval
query I con2
SELECT nextval('seq')
----
1
# query the sequence in con as well
query I
SELECT nextval('seq')
----
2
# restart the database
restart
# the sequence is now at position 3
query I
SELECT nextval('seq')
----
3

73
external/duckdb/test/sql/storage/catalog/test_storage_sequences.cpp vendored Normal file
View File

@@ -0,0 +1,73 @@
#include "catch.hpp"
#include "duckdb/common/file_system.hpp"
#include "test_helpers.hpp"
using namespace duckdb;
using namespace std;
TEST_CASE("Use sequences over different runs without checkpointing", "[storage]") {
duckdb::unique_ptr<QueryResult> result;
auto storage_database = TestCreatePath("storage_test");
// make sure the database does not exist
DeleteDatabase(storage_database);
{
// create a database and insert values
DuckDB db(storage_database);
Connection con(db);
REQUIRE_NO_FAIL(con.Query("CREATE SEQUENCE seq;"));
REQUIRE_NO_FAIL(con.Query("CREATE SEQUENCE seq_cycle INCREMENT 1 MAXVALUE 3 START 2 CYCLE;"));
result = con.Query("SELECT nextval('seq')");
REQUIRE(CHECK_COLUMN(result, 0, {1}));
result = con.Query("SELECT currval('seq')");
REQUIRE(CHECK_COLUMN(result, 0, {1}));
result = con.Query("SELECT currval('seq')");
REQUIRE(CHECK_COLUMN(result, 0, {1}));
result = con.Query("SELECT nextval('seq_cycle')");
REQUIRE(CHECK_COLUMN(result, 0, {2}));
}
// reload the database from disk twice
{
DuckDB db(storage_database);
Connection con(db);
}
{
DuckDB db(storage_database);
Connection con(db);
result = con.Query("SELECT nextval('seq')");
REQUIRE(CHECK_COLUMN(result, 0, {2}));
result = con.Query("SELECT currval('seq')");
REQUIRE(CHECK_COLUMN(result, 0, {2}));
result = con.Query("SELECT nextval('seq_cycle')");
REQUIRE(CHECK_COLUMN(result, 0, {3}));
result = con.Query("SELECT currval('seq_cycle')");
REQUIRE(CHECK_COLUMN(result, 0, {3}));
}
// reload again
{
DuckDB db(storage_database);
Connection con(db);
result = con.Query("SELECT nextval('seq'), nextval('seq');");
REQUIRE(CHECK_COLUMN(result, 0, {3}));
REQUIRE(CHECK_COLUMN(result, 1, {4}));
result = con.Query("SELECT nextval('seq_cycle')");
REQUIRE(CHECK_COLUMN(result, 0, {1}));
result = con.Query("SELECT currval('seq'), currval('seq_cycle')");
REQUIRE(CHECK_COLUMN(result, 0, {4}));
REQUIRE(CHECK_COLUMN(result, 1, {1}));
// drop sequence
REQUIRE_NO_FAIL(con.Query("DROP SEQUENCE seq;"));
}
{
// reload
DuckDB db(storage_database);
Connection con(db);
// the sequence is gone now
REQUIRE_FAIL(con.Query("SELECT nextval('seq')"));
// the other sequence is still there
REQUIRE_NO_FAIL(con.Query("SELECT nextval('seq_cycle')"));
}
DeleteDatabase(storage_database);
}

45
external/duckdb/test/sql/storage/catalog/test_store_add_column.test vendored Normal file
View File

@@ -0,0 +1,45 @@
# name: test/sql/storage/catalog/test_store_add_column.test
# description: Test storage of alter table add column
# group: [catalog]
load __TEST_DIR__/test_store_add_column.db
# create a table and add a column to it
statement ok
CREATE TABLE test (a INTEGER, b INTEGER);
statement ok
INSERT INTO test VALUES (11, 22), (13, 22), (12, 21)
statement ok
ALTER TABLE test ADD COLUMN k INTEGER DEFAULT 2
query I
SELECT k FROM test ORDER BY k
----
2
2
2
restart
# reload and verify the column is still there
query I
SELECT k FROM test ORDER BY k
----
2
2
2
# verify the default is still correct
statement ok
INSERT INTO test(a, b) VALUES (1, 1)
query I
SELECT k FROM test ORDER BY k
----
2
2
2
2

37
external/duckdb/test/sql/storage/catalog/test_store_add_column_persistent.test vendored Normal file
View File

@@ -0,0 +1,37 @@
# name: test/sql/storage/catalog/test_store_add_column_persistent.test
# description: Add column to persistent table
# group: [catalog]
load __TEST_DIR__/test_store_add_column_persistent.db
# first create a persistent table and insert data
statement ok
CREATE TABLE test (a INTEGER, b INTEGER);
statement ok
INSERT INTO test VALUES (11, 22), (13, 22), (12, 21)
# now reload
restart
# add the column
statement ok
ALTER TABLE test ADD COLUMN k INTEGER DEFAULT 2
query I
SELECT k FROM test ORDER BY k
----
2
2
2
# reload again
restart
# the column is still there
query I
SELECT k FROM test ORDER BY k
----
2
2
2

75
external/duckdb/test/sql/storage/catalog/test_store_alter_type.test vendored Normal file
View File

@@ -0,0 +1,75 @@
# name: test/sql/storage/catalog/test_store_alter_type.test
# description: Remove column from persistent table
# group: [catalog]
load __TEST_DIR__/test_store_alter_type.db
# create a table and add a column to it
statement ok
CREATE TABLE test (a INTEGER, b INTEGER);
statement ok
INSERT INTO test VALUES (11, 22), (13, 22), (12, 21)
restart
statement ok
PRAGMA enable_verification
statement ok
ALTER TABLE test ALTER b TYPE VARCHAR
query IT
SELECT * FROM test ORDER BY 1
----
11 22
12 21
13 22
statement ok
INSERT INTO test VALUES (10, 'hello')
query IT
SELECT * FROM test ORDER BY 1
----
10 hello
11 22
12 21
13 22
query IT
SELECT * FROM test WHERE b='hello'
----
10 hello
query I
DELETE FROM test WHERE b='hello'
----
1
query IT
SELECT * FROM test ORDER BY 1
----
11 22
12 21
13 22
restart
query IT
SELECT * FROM test ORDER BY 1
----
11 22
12 21
13 22
statement ok
INSERT INTO test VALUES (10, 'hello')
query IT
SELECT * FROM test ORDER BY 1
----
10 hello
11 22
12 21
13 22

45
external/duckdb/test/sql/storage/catalog/test_store_alter_type_crash.test vendored Normal file
View File

@@ -0,0 +1,45 @@
# name: test/sql/storage/catalog/test_store_alter_type_crash.test
# description: Remove column from persistent table
# group: [catalog]
load __TEST_DIR__/test_store_alter_type.db
# create a table and add a column to it
statement ok
CREATE TABLE test (a INTEGER, b INTEGER);
statement ok
INSERT INTO test VALUES (11, 22), (13, 22), (12, 21)
restart
statement ok
ALTER TABLE test ALTER b TYPE VARCHAR
query IT
SELECT * FROM test ORDER BY 1
----
11 22
12 21
13 22
statement ok
INSERT INTO test VALUES (10, 'hello')
query IT
SELECT * FROM test ORDER BY 1
----
10 hello
11 22
12 21
13 22
restart
query IT
SELECT * FROM test ORDER BY 1
----
10 hello
11 22
12 21
13 22

63
external/duckdb/test/sql/storage/catalog/test_store_default_sequence.test vendored Normal file
View File

@@ -0,0 +1,63 @@
# name: test/sql/storage/catalog/test_store_default_sequence.test
# description: Test storage of default values with sequences
# group: [catalog]
load __TEST_DIR__/test_store_defaults.db
# create a table with a reference to a sequence as default value
statement ok
CREATE SEQUENCE seq;
# use the sequence so that currval can return a value
query I
SELECT nextval('seq')
----
1
statement ok
CREATE TABLE test (a INTEGER DEFAULT nextval('seq'), b INTEGER, c INTEGER DEFAULT currval('seq'));
statement ok
INSERT INTO test (b) VALUES (11)
# restart the database
restart
query III
SELECT * FROM test ORDER BY b
----
2
11
2
# verify that the sequence was used during the append
statement ok
INSERT INTO test (b) VALUES (12);
statement ok
INSERT INTO test (b) VALUES (13);
query III
SELECT * FROM test ORDER BY b
----
2 11 2
3 12 3
4 13 4
# restart and insert one more time
restart
statement ok
INSERT INTO test (b) VALUES (14)
statement ok
INSERT INTO test (b) VALUES (15)
query III
SELECT * FROM test ORDER BY b
----
2 11 2
3 12 3
4 13 4
5 14 5
6 15 6

56
external/duckdb/test/sql/storage/catalog/test_store_defaults.test vendored Normal file
View File

@@ -0,0 +1,56 @@
# name: test/sql/storage/catalog/test_store_defaults.test
# description: Test storage of default values
# group: [catalog]
# load the DB from disk
load __TEST_DIR__/test_store_defaults.db
# create a table with a default value
statement ok
CREATE TABLE test (a INTEGER DEFAULT 1, b INTEGER);
statement ok
INSERT INTO test (b) VALUES (11)
query II
SELECT * FROM test ORDER BY b
----
1 11
# reload the database
restart
# verify that the table contains the correct contents
query II
SELECT * FROM test ORDER BY b
----
1 11
# append more entries
statement ok
INSERT INTO test (b) VALUES (12), (13)
# check that the default value was used in the INSERT
query II
SELECT * FROM test ORDER BY b
----
1 11
1 12
1 13
# reload and append one more time
restart
# append more entries
statement ok
INSERT INTO test (b) VALUES (14), (15)
# check that the default value was used in the INSERT
query II
SELECT * FROM test ORDER BY b
----
1 11
1 12
1 13
1 14
1 15

33
external/duckdb/test/sql/storage/catalog/test_store_remove_column.test vendored Normal file
View File

@@ -0,0 +1,33 @@
# name: test/sql/storage/catalog/test_store_remove_column.test
# description: Remove column from persistent table
# group: [catalog]
load __TEST_DIR__/test_store_add_column.db
# create a table and add a column to it
statement ok
CREATE TABLE test (a INTEGER, b INTEGER);
statement ok
INSERT INTO test VALUES (11, 22), (13, 22), (12, 21)
restart
statement ok
ALTER TABLE test DROP COLUMN b
query I
SELECT * FROM test ORDER BY 1
----
11
12
13
restart
query I
SELECT * FROM test ORDER BY 1
----
11
12
13

80
external/duckdb/test/sql/storage/catalog/test_store_rename_column.test vendored Normal file
View File

@@ -0,0 +1,80 @@
# name: test/sql/storage/catalog/test_store_rename_column.test
# description: Test storage of alter table rename column
# group: [catalog]
# load the DB from disk
load __TEST_DIR__/test_rename_column.db
statement ok
CREATE TABLE test (a INTEGER, b INTEGER);
statement ok
INSERT INTO test VALUES (11, 22), (13, 22), (12, 21)
# rename the column and rollback the transaction
statement ok
BEGIN TRANSACTION
query I
SELECT a FROM test ORDER BY a
----
11
12
13
statement ok
ALTER TABLE test RENAME COLUMN a TO k
query I
SELECT k FROM test ORDER BY k
----
11
12
13
statement ok
ROLLBACK
# reload
restart
statement ok
BEGIN TRANSACTION
# verify that the column is not renamed
query I
SELECT a FROM test ORDER BY a
----
11
12
13
# now repeat the process but this time commit
statement ok
ALTER TABLE test RENAME COLUMN a TO k
query I
SELECT k FROM test ORDER BY k
----
11
12
13
statement ok
COMMIT
# reload the database
restart
# verify that the column is still renamed
query I
SELECT k FROM test ORDER BY k
----
11
12
13
statement error
SELECT a FROM test
----
<REGEX>:.*Binder Error.*not found in FROM clause.*

73
external/duckdb/test/sql/storage/catalog/test_store_rename_table.test vendored Normal file
View File

@@ -0,0 +1,73 @@
# name: test/sql/storage/catalog/test_store_rename_table.test
# description: Test storage of alter table rename table
# group: [catalog]
# load the DB from disk
load __TEST_DIR__/test_rename_table.db
statement ok
CREATE TABLE test (a INTEGER, b INTEGER);
statement ok
INSERT INTO test VALUES (11, 22), (13, 22), (12, 21)
statement ok
BEGIN TRANSACTION
query I
SELECT a FROM test ORDER BY a
----
11
12
13
statement ok
ALTER TABLE test RENAME TO new_name
query I
SELECT a FROM new_name ORDER BY 1
----
11
12
13
statement ok
ROLLBACK
# restart the database
restart
statement ok
BEGIN TRANSACTION
# verify that the table is still there in the original form
query I
SELECT a FROM test ORDER BY a
----
11
12
13
# now repeat the process, but this time commit
statement ok
ALTER TABLE test RENAME TO new_name
query I
SELECT a FROM new_name ORDER BY 1
----
11
12
13
statement ok
COMMIT
restart
# after a restart, the renamed table is still here
query I
SELECT a FROM new_name ORDER BY 1
----
11
12
13

76
external/duckdb/test/sql/storage/catalog/test_store_rename_view.test vendored Normal file
View File

@@ -0,0 +1,76 @@
# name: test/sql/storage/catalog/test_store_rename_view.test
# description: Test storage of alter view
# group: [catalog]
# load the DB from disk
load __TEST_DIR__/test_rename_view.db
statement ok
CREATE TABLE test (a INTEGER, b INTEGER);
statement ok
INSERT INTO test VALUES (11, 22), (13, 22), (12, 21)
statement ok
CREATE VIEW vtest AS SELECT * FROM test
statement ok
BEGIN TRANSACTION
query I
SELECT a FROM vtest ORDER BY a
----
11
12
13
statement ok
ALTER VIEW vtest RENAME TO new_name
query I
SELECT a FROM new_name ORDER BY 1
----
11
12
13
statement ok
ROLLBACK
# restart the database
restart
statement ok
BEGIN TRANSACTION
# verify that the table is still there in the original form
query I
SELECT a FROM vtest ORDER BY a
----
11
12
13
# now repeat the process, but this time commit
statement ok
ALTER VIEW vtest RENAME TO new_name
query I
SELECT a FROM new_name ORDER BY 1
----
11
12
13
statement ok
COMMIT
restart
# after a restart, the renamed table is still here
query I
SELECT a FROM new_name ORDER BY 1
----
11
12
13

64
external/duckdb/test/sql/storage/catalog/test_store_sequences.test vendored Normal file
View File

@@ -0,0 +1,64 @@
# name: test/sql/storage/catalog/test_store_sequences.test
# description: Use sequences over different runs
# group: [catalog]
load __TEST_DIR__/store_sequences.db
# standard sequence
statement ok
CREATE SEQUENCE seq
# more complex sequence
statement ok
CREATE SEQUENCE seq_cycle INCREMENT 1 MAXVALUE 3 START 2 CYCLE;
query I
SELECT nextval('seq')
----
1
query I
SELECT nextval('seq_cycle')
----
2
# restart and check that sequence still works
restart
query I
SELECT nextval('seq')
----
2
query I
SELECT nextval('seq_cycle')
----
3
# again
restart
query II
SELECT nextval('seq'), nextval('seq')
----
3 4
query I
SELECT nextval('seq_cycle')
----
1
# drop sequence
statement ok
DROP SEQUENCE seq;
restart
# verify that sequence is gone
statement error
SELECT nextval('seq')
----
# other sequence is still there
statement ok
SELECT nextval('seq_cycle')

88
external/duckdb/test/sql/storage/catalog/test_store_temporary.test vendored Normal file
View File

@@ -0,0 +1,88 @@
# name: test/sql/storage/catalog/test_store_temporary.test
# description: Temporary tables are not written to disk
# group: [catalog]
# load the DB from disk
load __TEST_DIR__/test_store_temporary.db
# create a temporary table and do some ops on there
statement ok
CREATE TABLE persistent (i INTEGER)
statement ok
CREATE TEMPORARY TABLE temp.a (i INTEGER)
# insert values, delete them, etc
statement ok
INSERT INTO a VALUES (42)
statement ok
DELETE FROM a
statement ok
DELETE FROM temp.a
# check schema
statement error
DELETE FROM asdf.a
----
# temporary sequences
statement ok
CREATE TEMPORARY SEQUENCE seq
statement ok
CREATE TEMPORARY SEQUENCE seq2
statement ok
DROP SEQUENCE seq2
# temporary views
statement ok
CREATE TEMPORARY VIEW v1 AS SELECT 42
statement ok
CREATE TEMPORARY VIEW v2 AS SELECT 42
statement ok
DROP VIEW v2
statement ok
INSERT INTO temp.a VALUES (43)
statement ok
UPDATE temp.a SET i = 44
statement ok
UPDATE a SET i = 45
# alter table on a temporary structure
statement ok
ALTER TABLE a RENAME COLUMN i TO k
statement ok
SELECT * FROM persistent
statement ok
SELECT * FROM a
# now restart
restart
# persistent table is still there
statement ok
SELECT * FROM persistent
# temporary table is gone
statement error
SELECT * FROM a
----
statement ok
CREATE TEMPORARY TABLE a (i INTEGER)
statement ok
CREATE TEMPORARY SEQUENCE seq
statement ok
CREATE TEMPORARY VIEW v1 AS SELECT 42

91
external/duckdb/test/sql/storage/catalog/test_table_macro_storage.test vendored Normal file
View File

@@ -0,0 +1,91 @@
# name: test/sql/storage/catalog/test_table_macro_storage.test
# description: Test storage of table macros
# group: [catalog]
require skip_reload
# load the DB from disk
load __TEST_DIR__/macro_storage.db
statement ok
CREATE TABLE test_tbl (id INT, name string, height double);
statement ok
INSERT INTO test_tbl values (1,'tom', 1.1), (2,'dick',1.2),(3,'harry', 1.2),
(4,'mary',0.9), (5,'mungo', 0.8), (6,'midge', 0.5);
# create a table macro
statement ok
CREATE MACRO xt(a, _name) as TABLE SELECT * FROM test_tbl WHERE id<=a or name = _name;
# use the macro
query III
SELECT * FROM xt(10, '*') ORDER BY height limit 1;
----
6 midge 0.5
statement ok
CREATE TEMPORARY MACRO my_seq(start , finish, stride:=3) as TABLE SELECT * FROM generate_series(start , finish , stride);
query I
SELECT * FROM my_seq(0,6);
----
0
3
6
restart
statement ok
PRAGMA disable_checkpoint_on_shutdown
statement ok
PRAGMA wal_autocheckpoint='1TB';
statement ok
SELECT * FROM xt(100, 'joe');
statement ok
DROP MACRO TABLE xt;
statement error
SELECT * from my_seq(0,10,2);
----
# create another macro to check the WAL
statement ok
CREATE MACRO my_range(rend) AS TABLE SELECT * FROM range(rend);
query I
SELECT * from my_range(2);
----
0
1
# check the WAL
restart
statement error
SELECT * FROM xt(100, 'joe');
----
statement error
SELECT * FROM my_seq(0,3,2);
----
query I
SELECT * from my_range(2);
----
0
1

72
external/duckdb/test/sql/storage/catalog/test_view_explicit_aliases.test vendored Normal file
View File

@@ -0,0 +1,72 @@
# name: test/sql/storage/catalog/test_view_explicit_aliases.test
# description: Test views with explicit column aliases
# group: [catalog]
# FIXME: for this to work we need to serialize/deserialize dependencies
require skip_reload
# load the DB from disk
load __TEST_DIR__/view_explicit_aliases_storage.db
statement ok
set enable_view_dependencies=true
# create a database and insert the table/view
statement ok
CREATE SCHEMA test;
statement ok
CREATE TABLE test.t (a INTEGER, b INTEGER);
# the view has aliases (b, c)
statement ok
CREATE VIEW test.v (b,c) AS SELECT * FROM test.t;
# check the view info
query IIIIII
PRAGMA table_info('test.v')
----
0 b INTEGER false NULL false
1 c INTEGER false NULL false
statement ok
SELECT * FROM test.v
statement ok
DROP TABLE test.t CASCADE;
statement error
PRAGMA table_info('test.v')
----
loop i 0 2
# now reload
restart
statement error
PRAGMA table_info('test.v')
----
statement error
SELECT * FROM test.v
----
# we can query again after recreating the table
statement ok
CREATE TABLE test.t (a INTEGER, b INTEGER);
statement ok
SELECT * FROM test.t
statement error
SELECT b,c FROM test.v
----
statement ok
CREATE VIEW test.v (b,c) AS SELECT * FROM test.t;
statement ok
DROP TABLE test.t CASCADE;
endloop

85
external/duckdb/test/sql/storage/catalog/test_view_storage.test vendored Normal file
View File

@@ -0,0 +1,85 @@
# name: test/sql/storage/catalog/test_view_storage.test
# description: Create and drop a view over different runs
# group: [catalog]
# FIXME: for this to work we need to serialize/deserialize dependencies
require skip_reload
# load the DB from disk
load __TEST_DIR__/view_storage.db
statement ok
set enable_view_dependencies=true
# create a schema and view
statement ok
CREATE SCHEMA test;
statement ok
CREATE TABLE test.t (a INTEGER, b INTEGER);
statement ok
CREATE VIEW test.v AS SELECT * FROM test.t;
# read the info from the view
query IIIIII
PRAGMA table_info('test.v')
----
0 a INTEGER 0 NULL 0
1 b INTEGER 0 NULL 0
# drop the table the view is based on
statement ok
DROP TABLE test.t CASCADE;
statement error
PRAGMA table_info('test.v')
----
# but querying the view fails
statement error
SELECT * FROM test.v
----
loop i 0 2
# restart the system
restart
statement error
PRAGMA table_info('test.v')
----
statement error
SELECT * FROM test.v
----
# after recreating the table, we can query the view again
statement ok
CREATE TABLE test.t (a INTEGER, b INTEGER);
statement ok
SELECT * FROM test.t
statement error
SELECT * FROM test.v
----
statement ok
CREATE VIEW test.v AS SELECT * FROM test.t;
statement ok
SELECT * FROM test.v
query IIIIII
PRAGMA table_info('test.v')
----
0 a INTEGER 0 NULL 0
1 b INTEGER 0 NULL 0
# drop the table again
statement ok
DROP TABLE test.t CASCADE;
endloop

79
external/duckdb/test/sql/storage/catalog/test_view_storage_no_view_dependencies.test vendored Normal file
View File

@@ -0,0 +1,79 @@
# name: test/sql/storage/catalog/test_view_storage_no_view_dependencies.test
# description: Create and drop a view over different runs
# group: [catalog]
# load the DB from disk
load __TEST_DIR__/view_storage.db
# create a schema and view
statement ok
CREATE SCHEMA test;
statement ok
CREATE TABLE test.t (a INTEGER, b INTEGER);
statement ok
CREATE VIEW test.v AS SELECT * FROM test.t;
# read the info from the view
query IIIIII
PRAGMA table_info('test.v')
----
0 a INTEGER 0 NULL 0
1 b INTEGER 0 NULL 0
# drop the table the view is based on
statement ok
DROP TABLE test.t
# we can still query the types and column names
query IIIIII
PRAGMA table_info('test.v')
----
0 a INTEGER 0 NULL 0
1 b INTEGER 0 NULL 0
# but querying the view fails
statement error
SELECT * FROM test.v
----
loop i 0 2
# restart the system
restart
# the view still exists, but the table does not
# we can check the types, but not query it
query IIIIII
PRAGMA table_info('test.v')
----
0 a INTEGER 0 NULL 0
1 b INTEGER 0 NULL 0
statement error
SELECT * FROM test.v
----
# after recreating the table, we can query the view again
statement ok
CREATE TABLE test.t (a INTEGER, b INTEGER);
statement ok
SELECT * FROM test.t
statement ok
SELECT * FROM test.v
query IIIIII
PRAGMA table_info('test.v')
----
0 a INTEGER 0 NULL 0
1 b INTEGER 0 NULL 0
# drop the table again
statement ok
DROP TABLE test.t
endloop

51
external/duckdb/test/sql/storage/checkpoint/concurrent_load_delete.test_slow vendored Normal file
View File

@@ -0,0 +1,51 @@
# name: test/sql/storage/checkpoint/concurrent_load_delete.test_slow
# description: Test concurrent delete load workflow
# group: [checkpoint]
load __TEST_DIR__/concurrent_delete_load.db
statement ok
create or replace table z(id integer);
statement ok
insert into z from range(10_000_000);
loop i 0 100
statement ok
PRAGMA disable_checkpoint_on_shutdown
statement ok
SET checkpoint_threshold='1TB'
concurrentloop c 0 7
onlyif c=0
statement ok
FORCE CHECKPOINT
onlyif c>0&&c<=5
statement ok
SELECT SUM(id) FROM z
onlyif c=5
statement ok
DELETE FROM z WHERE id%((random() * 3)::UBIGINT)=0
onlyif c=6
statement ok
INSERT INTO z FROM range(1000, 100000 + ${c} * 100000)
endloop
restart
endloop
statement ok
CHECKPOINT
restart
statement ok
SELECT SUM(id) FROM z

39
external/duckdb/test/sql/storage/checkpoint/keep_small_row_groups.test_slow vendored Normal file
View File

@@ -0,0 +1,39 @@
# name: test/sql/storage/checkpoint/keep_small_row_groups.test_slow
# description: Test keeping small row groups
# group: [checkpoint]
load __TEST_DIR__/keep_small_row_groups.db
foreach type noindex index
onlyif type=noindex
statement ok
create or replace table z(id integer, a varchar, b varchar, c varchar);
onlyif type=index
statement ok
create or replace table z(id integer primary key, a varchar, b varchar, c varchar);
statement ok
insert into z select i, sha256(i::varchar) as a, sha256((i**2)::varchar) as b, sha256((i**3)::varchar) as c from range(100000) r(i);
statement ok
CHECKPOINT;
# we insert 100K rows, as part of small 100 row inserts, and checkpoint for each iteration
loop i 0 100
statement ok
insert into z select 100000 + 100 * ${i} + i, 'a','b','c' from range(100) t(i);
statement ok
CHECKPOINT;
endloop
query I
SELECT COUNT(DISTINCT row_group_id) < 5 FROM pragma_storage_info('z')
----
true
endloop

87
external/duckdb/test/sql/storage/checkpoint_abort.test_slow vendored Normal file
View File

@@ -0,0 +1,87 @@
# name: test/sql/storage/checkpoint_abort.test_slow
# description: Test correct behavior if we unexpectedly abort during a checkpoint
# group: [storage]
load __TEST_DIR__/checkpoint_abort.db
statement ok
CREATE TABLE integers AS SELECT * FROM range(100000) tbl(i);
statement ok
CHECKPOINT;
statement ok
PRAGMA disable_checkpoint_on_shutdown;
statement ok
PRAGMA wal_autocheckpoint='1TB';
statement ok
PRAGMA debug_checkpoint_abort='before_header'
query III
SELECT MIN(i), MAX(i), COUNT(*) FROM integers
----
0 99999 100000
statement ok
INSERT INTO integers SELECT * FROM range(100000, 200000) tbl(i);
query III
SELECT MIN(i), MAX(i), COUNT(*) FROM integers
----
0 199999 200000
statement ok
UPDATE integers SET i=i+1;
query III
SELECT MIN(i), MAX(i), COUNT(*) FROM integers
----
1 200000 200000
statement error
CHECKPOINT;
----
restart
# verify that the change was correctly loaded from disk
query III
SELECT MIN(i), MAX(i), COUNT(*) FROM integers
----
1 200000 200000
# now verify that empty blocks left by a checkpoint aborts are re-used
# so that checkpoint aborts don't permanently leave holes in the file
loop i 0 10
statement ok
PRAGMA disable_checkpoint_on_shutdown;
statement ok
PRAGMA wal_autocheckpoint='1TB';
statement ok
PRAGMA debug_checkpoint_abort='before_header'
statement ok
UPDATE integers SET i=i;
statement error
CHECKPOINT;
----
restart
# verify that the change was correctly loaded from disk
query III
SELECT MIN(i), MAX(i), COUNT(*) FROM integers
----
1 200000 200000
query I nosort expected_blocks
select total_blocks from pragma_database_size();
endloop

71
external/duckdb/test/sql/storage/checkpoint_abort_after_free_list.test_slow vendored Normal file
View File

@@ -0,0 +1,71 @@
# name: test/sql/storage/checkpoint_abort_after_free_list.test_slow
# description: Test correct behavior if we unexpectedly abort after a checkpoint right after the free list is written
# group: [storage]
load __TEST_DIR__/checkpoint_abort.db
statement ok
CREATE TABLE integers AS SELECT * FROM range(100000) tbl(i);
statement ok
CHECKPOINT;
statement ok
PRAGMA disable_checkpoint_on_shutdown;
statement ok
PRAGMA wal_autocheckpoint='1TB';
statement ok
PRAGMA debug_checkpoint_abort='after_free_list_write';
statement ok
INSERT INTO integers SELECT * FROM range(100000, 200000) tbl(i);
statement error
CHECKPOINT;
----
restart
# verify that the change was correctly loaded from disk
query III
SELECT MIN(i), MAX(i), COUNT(*) FROM integers;
----
0 199999 200000
# now verify that empty blocks left by a checkpoint aborts are re-used
# so that checkpoint aborts don't permanently leave holes in the file
loop i 0 10
statement ok
PRAGMA disable_checkpoint_on_shutdown;
statement ok
PRAGMA wal_autocheckpoint='1TB';
statement ok
PRAGMA debug_checkpoint_abort='after_free_list_write';
statement ok
UPDATE integers SET i=i;
statement error
CHECKPOINT;
----
restart
# verify that the change was correctly loaded from disk
query III
SELECT MIN(i), MAX(i), COUNT(*) FROM integers;
----
0 199999 200000
# ensure that the expected total storage size is the same as in the first iteration of the loop
query I nosort expected_blocks
SELECT total_blocks FROM pragma_database_size();
endloop

73
external/duckdb/test/sql/storage/checkpoint_abort_before_header.test_slow vendored Normal file
View File

@@ -0,0 +1,73 @@
# name: test/sql/storage/checkpoint_abort_before_header.test_slow
# description: Test correct behavior if we unexpectedly abort after a checkpoint but before the WAL is successfully truncated
# group: [storage]
load __TEST_DIR__/checkpoint_abort.db
statement ok
CREATE TABLE integers AS SELECT * FROM range(100000) tbl(i);
statement ok
CHECKPOINT;
statement ok
PRAGMA disable_checkpoint_on_shutdown;
statement ok
PRAGMA wal_autocheckpoint='1TB';
statement ok
PRAGMA debug_checkpoint_abort='before_header';
statement ok
INSERT INTO integers SELECT * FROM range(100000, 200000) tbl(i);
statement error
CHECKPOINT;
----
Checkpoint aborted before header write
restart
# verify that the change was correctly loaded from disk
query III
SELECT MIN(i), MAX(i), COUNT(*) FROM integers;
----
0 199999 200000
# now verify that empty blocks left by a checkpoint abort are re-used
# so that checkpoint aborts don't permanently leave holes in the file
loop i 0 50
statement ok
PRAGMA disable_checkpoint_on_shutdown;
statement ok
PRAGMA wal_autocheckpoint='1TB';
statement ok
PRAGMA debug_checkpoint_abort='before_header';
statement ok
UPDATE integers SET i=i;
statement error
CHECKPOINT;
----
Checkpoint aborted before header write
restart
# verify that the change was correctly loaded from disk
query III
SELECT MIN(i), MAX(i), COUNT(*) FROM integers;
----
0 199999 200000
# ensure that the expected total storage size is the same as in the first iteration of the loop
query I nosort expected_blocks
SELECT total_blocks FROM pragma_database_size();
endloop

74
external/duckdb/test/sql/storage/checkpoint_abort_before_truncate.test_slow vendored Normal file
View File

@@ -0,0 +1,74 @@
# name: test/sql/storage/checkpoint_abort_before_truncate.test_slow
# description: Test correct behavior if we unexpectedly abort after a checkpoint but before the WAL is successfully truncated
# group: [storage]
load __TEST_DIR__/checkpoint_abort.db
statement ok
CREATE TABLE integers AS SELECT * FROM range(100000) tbl(i);
statement ok
CHECKPOINT;
statement ok
PRAGMA disable_checkpoint_on_shutdown;
statement ok
PRAGMA wal_autocheckpoint='1TB';
statement ok
PRAGMA debug_checkpoint_abort='before_truncate';
statement ok
INSERT INTO integers SELECT * FROM range(100000, 200000) tbl(i);
statement error
CHECKPOINT;
----
restart
# verify that the change was correctly loaded from disk
query III
SELECT MIN(i), MAX(i), COUNT(*) FROM integers;
----
0 199999 200000
# now verify that empty blocks left by a checkpoint aborts are re-used
# so that checkpoint aborts don't permanently leave holes in the file
loop i 0 100
statement ok
PRAGMA disable_checkpoint_on_shutdown;
statement ok
PRAGMA wal_autocheckpoint='1TB';
statement ok
PRAGMA debug_checkpoint_abort='before_truncate';
statement ok
UPDATE integers SET i=i+1;
statement ok
UPDATE integers SET i=i-1;
statement error
CHECKPOINT;
----
restart
# verify that the change was correctly loaded from disk
query III
SELECT MIN(i), MAX(i), COUNT(*) FROM integers;
----
0 199999 200000
# ensure that the expected total storage size is the same as in the first iteration of the loop
query I nosort expected_blocks
SELECT total_blocks FROM pragma_database_size();
endloop

39
external/duckdb/test/sql/storage/checkpoint_exactly_morsel_size.test_slow vendored Normal file
View File

@@ -0,0 +1,39 @@
# name: test/sql/storage/checkpoint_exactly_morsel_size.test_slow
# description: Run a checkpoint on a table that is exactly morsel size
# group: [storage]
load __TEST_DIR__/checkpoint_morsel_size.db
statement ok
CREATE TABLE integers AS SELECT * FROM range(1024*100*8) tbl(i);
statement ok
CHECKPOINT;
query III
SELECT min(i), max(i), count(i) from integers;
----
0 819199 819200
statement ok
UPDATE integers SET i=i+1;
query III
SELECT min(i), max(i), count(i) from integers;
----
1 819200 819200
restart
query III
SELECT min(i), max(i), count(i) from integers;
----
1 819200 819200
statement ok
DELETE FROM integers WHERE i<=1000
query III
SELECT min(i), max(i), count(i) from integers;
----
1001 819200 818200

55
external/duckdb/test/sql/storage/checkpoint_with_pending_updates.test_slow vendored Normal file
View File

@@ -0,0 +1,55 @@
# name: test/sql/storage/checkpoint_with_pending_updates.test_slow
# description: Test checkpoint with pending updates
# group: [storage]
# load the DB from disk
load __TEST_DIR__/pending_updates.db
statement ok
CREATE TABLE test (i INTEGER);
statement ok
INSERT INTO test SELECT * FROM range(1000000);
statement ok
BEGIN TRANSACTION;
statement ok
UPDATE test SET i=i+1;
# cannot checkpoint: this transaction has transaction-local changes
statement error
CHECKPOINT
----
statement ok
ROLLBACK
# now we can checkpoint
statement ok
CHECKPOINT
statement ok
BEGIN TRANSACTION;
statement ok
UPDATE test SET i=i+1;
# We ALSO cannot force checkpoint when we have transaction local changes ourselves (can only abort OTHER transactions)
statement error
FORCE CHECKPOINT
----
statement ok
ROLLBACK
# now we can checkpoint
statement ok
CHECKPOINT
restart
query III
SELECT MIN(i), MAX(i), COUNT(*) FROM test;
----
0 999999 1000000

39
external/duckdb/test/sql/storage/checkpointed_self_append.test vendored Normal file
View File

@@ -0,0 +1,39 @@
# name: test/sql/storage/checkpointed_self_append.test
# description: Test appending to a checkpointed table from itself
# group: [storage]
# load the DB from disk
load __TEST_DIR__/checkpointed_self_append.db
statement ok
PRAGMA disable_checkpoint_on_shutdown
statement ok
PRAGMA wal_autocheckpoint='1TB';
statement ok
CREATE TABLE vals(i INTEGER)
statement ok
INSERT INTO vals SELECT CASE WHEN i % 2 = 0 THEN NULL ELSE i END FROM range(200000) tbl(i)
query IIII
SELECT MIN(i), MAX(i), COUNT(i), COUNT(*) FROM vals
----
1 199999 100000 200000
statement ok
CHECKPOINT
query IIII
SELECT MIN(i), MAX(i), COUNT(i), COUNT(*) FROM vals
----
1 199999 100000 200000
statement ok
INSERT INTO vals SELECT * FROM vals;
query IIII
SELECT MIN(i), MAX(i), COUNT(i), COUNT(*) FROM vals
----
1 199999 200000 400000

35
external/duckdb/test/sql/storage/checkpointed_self_append_tinyint.test vendored Normal file
View File

@@ -0,0 +1,35 @@
# name: test/sql/storage/checkpointed_self_append_tinyint.test
# description: Test appending to a checkpointed table from itself with tinyint columns
# group: [storage]
# load the DB from disk
load __TEST_DIR__/checkpointed_self_append.db
statement ok
CREATE TABLE vals(i TINYINT)
statement ok
INSERT INTO vals SELECT (CASE WHEN i % 2 = 0 THEN NULL ELSE i % 100 END)::TINYINT i FROM range(200000) tbl(i)
query IIII
SELECT MIN(i), MAX(i), COUNT(i), COUNT(*) FROM vals
----
1 99 100000 200000
statement ok
CHECKPOINT
query IIII
SELECT MIN(i), MAX(i), COUNT(i), COUNT(*) FROM vals
----
1 99 100000 200000
query I
INSERT INTO vals SELECT * FROM vals;
----
200000
query IIII
SELECT MIN(i), MAX(i), COUNT(i), COUNT(*) FROM vals
----
1 99 200000 400000

100
external/duckdb/test/sql/storage/commit_abort.test vendored Normal file
View File

@@ -0,0 +1,100 @@
# name: test/sql/storage/commit_abort.test
# description: Test abort of commit with persistent storage
# group: [storage]
# load the DB from disk
load __TEST_DIR__/commit_abort.db
statement ok
PRAGMA enable_verification
statement ok
CREATE TABLE test (a INTEGER PRIMARY KEY, b INTEGER, c VARCHAR);
statement ok
INSERT INTO test VALUES (11, 22, 'hello'), (13, 22, 'world'), (12, 21, 'test'), (10, NULL, NULL);
# start a transaction for con and con2
statement ok con1
BEGIN TRANSACTION
statement ok con2
BEGIN TRANSACTION
# insert the value 14 in both transactions
statement ok con1
INSERT INTO test VALUES (14, 10, 'con')
statement ok con2
INSERT INTO test VALUES (15, 10, 'con2')
statement ok con2
INSERT INTO test VALUES (14, 10, 'con2')
# commit both
# con2 will fail
statement ok con1
COMMIT
statement error con2
COMMIT
----
statement ok con1
INSERT INTO test VALUES (15, NULL, NULL)
query IIIIII
SELECT COUNT(*), COUNT(a), COUNT(b), SUM(a), SUM(b), SUM(LENGTH(c)) FROM test
----
6 6 4 75 75 17
query III
SELECT * FROM test ORDER BY a, b, c
----
10 NULL NULL
11 22 hello
12 21 test
13 22 world
14 10 con
15 NULL NULL
restart
statement ok
PRAGMA enable_verification
query III
SELECT * FROM test ORDER BY a, b, c
----
10 NULL NULL
11 22 hello
12 21 test
13 22 world
14 10 con
15 NULL NULL
query IIIIII
SELECT COUNT(*), COUNT(a), COUNT(b), SUM(a), SUM(b), SUM(LENGTH(c)) FROM test
----
6 6 4 75 75 17
restart
statement ok
PRAGMA enable_verification
query IIIIII
SELECT COUNT(*), COUNT(a), COUNT(b), SUM(a), SUM(b), SUM(LENGTH(c)) FROM test
----
6 6 4 75 75 17
query III
SELECT * FROM test ORDER BY a, b, c
----
10 NULL NULL
11 22 hello
12 21 test
13 22 world
14 10 con
15 NULL NULL

102
external/duckdb/test/sql/storage/commit_abort_large.test_slow vendored Normal file
View File

@@ -0,0 +1,102 @@
# name: test/sql/storage/commit_abort_large.test_slow
# description: Test abort of commit with many values
# group: [storage]
# load the DB from disk
load __TEST_DIR__/commit_abort.db
statement ok
PRAGMA wal_autocheckpoint='1TB';
statement ok
PRAGMA enable_verification
statement ok
CREATE TABLE test (a INTEGER PRIMARY KEY, b INTEGER, c VARCHAR);
statement ok
INSERT INTO test VALUES (11, 22, 'hello'), (13, 22, 'world'), (12, 21, 'test'), (10, NULL, NULL);
# start a transaction for con and con2
statement ok con1
BEGIN TRANSACTION
statement ok con2
BEGIN TRANSACTION
# insert the value 14 in con1
statement ok con1
INSERT INTO test VALUES (14, 10, 'con')
# insert a bunch of unique values in con2
statement ok con2
INSERT INTO test SELECT i, NULL, NULL FROM range(15, 1000000) tbl(i);
# now insert value 14
statement ok con2
INSERT INTO test VALUES (14, 10, 'con2')
# commit both
# con2 will fail
statement ok con1
COMMIT
statement error con2
COMMIT
----
query III
SELECT * FROM test ORDER BY a, b, c
----
10 NULL NULL
11 22 hello
12 21 test
13 22 world
14 10 con
statement ok con1
INSERT INTO test VALUES (15, NULL, NULL)
statement ok con1
INSERT INTO test VALUES (16, 24, 'blabla')
query III
SELECT * FROM test ORDER BY a, b, c
----
10 NULL NULL
11 22 hello
12 21 test
13 22 world
14 10 con
15 NULL NULL
16 24 blabla
restart
statement ok
PRAGMA enable_verification
query III
SELECT * FROM test ORDER BY a, b, c
----
10 NULL NULL
11 22 hello
12 21 test
13 22 world
14 10 con
15 NULL NULL
16 24 blabla
restart
query III
SELECT * FROM test ORDER BY a, b, c
----
10 NULL NULL
11 22 hello
12 21 test
13 22 world
14 10 con
15 NULL NULL
16 24 blabla

99
external/duckdb/test/sql/storage/commit_abort_medium.test vendored Normal file
View File

@@ -0,0 +1,99 @@
# name: test/sql/storage/commit_abort_medium.test
# description: Test abort of commit with many values
# group: [storage]
# load the DB from disk
load __TEST_DIR__/commit_abort.db
statement ok
PRAGMA enable_verification
statement ok
CREATE TABLE test (a INTEGER PRIMARY KEY, b INTEGER, c VARCHAR);
statement ok
INSERT INTO test VALUES (11, 22, 'hello'), (13, 22, 'world'), (12, 21, 'test'), (10, NULL, NULL);
# start a transaction for con and con2
statement ok con1
BEGIN TRANSACTION
statement ok con2
BEGIN TRANSACTION
# insert the value 14 in con1
statement ok con1
INSERT INTO test VALUES (14, 10, 'con')
# insert a bunch of unique values in con2
statement ok con2
INSERT INTO test SELECT i, NULL, NULL FROM range(15, 10000) tbl(i);
# now insert value 14
statement ok con2
INSERT INTO test VALUES (14, 10, 'con2')
# commit both
# con2 will fail
statement ok con1
COMMIT
statement error con2
COMMIT
----
query III
SELECT * FROM test ORDER BY a, b, c
----
10 NULL NULL
11 22 hello
12 21 test
13 22 world
14 10 con
statement ok con1
INSERT INTO test VALUES (15, NULL, NULL)
statement ok con1
INSERT INTO test VALUES (16, 24, 'blabla')
query III
SELECT * FROM test ORDER BY a, b, c
----
10 NULL NULL
11 22 hello
12 21 test
13 22 world
14 10 con
15 NULL NULL
16 24 blabla
restart
statement ok
PRAGMA enable_verification
query III
SELECT * FROM test ORDER BY a, b, c
----
10 NULL NULL
11 22 hello
12 21 test
13 22 world
14 10 con
15 NULL NULL
16 24 blabla
restart
query III
SELECT * FROM test ORDER BY a, b, c
----
10 NULL NULL
11 22 hello
12 21 test
13 22 world
14 10 con
15 NULL NULL
16 24 blabla

66
external/duckdb/test/sql/storage/commit_index_deletes.test vendored Normal file
View File

@@ -0,0 +1,66 @@
# name: test/sql/storage/commit_index_deletes.test
# description: Test commit of index with deletes
# group: [storage]
# load the DB from disk
load __TEST_DIR__/commit_abort.db
statement ok
PRAGMA enable_verification
statement ok
CREATE TABLE test (a INTEGER PRIMARY KEY, b INTEGER, c VARCHAR);
statement ok
INSERT INTO test VALUES (11, 22, 'hello'), (13, 22, 'world'), (12, 21, 'test'), (10, NULL, NULL);
statement ok
INSERT INTO test VALUES (14, 10, 'con')
query III
SELECT * FROM test ORDER BY a, b, c
----
10 NULL NULL
11 22 hello
12 21 test
13 22 world
14 10 con
# delete the value again
statement ok
DELETE FROM test WHERE a=14
query III
SELECT * FROM test ORDER BY a, b, c
----
10 NULL NULL
11 22 hello
12 21 test
13 22 world
# now add the value back in, slightly differently
statement ok
INSERT INTO test VALUES (14, 11, 'bla')
query III
SELECT * FROM test ORDER BY a, b, c
----
10 NULL NULL
11 22 hello
12 21 test
13 22 world
14 11 bla
restart
statement ok
PRAGMA enable_verification
query III
SELECT * FROM test ORDER BY a, b, c
----
10 NULL NULL
11 22 hello
12 21 test
13 22 world
14 11 bla

21
external/duckdb/test/sql/storage/compact_block_size/block_size_with_rollback.test vendored Normal file
View File

@@ -0,0 +1,21 @@
# name: test/sql/storage/compact_block_size/block_size_with_rollback.test
# description: Tests rolling back after an attach and then attaching the same file with a different block size.
# group: [compact_block_size]
statement ok
BEGIN TRANSACTION;
statement ok
ATTACH '__TEST_DIR__/rollback.db' (BLOCK_SIZE 16384);
statement ok
CREATE TABLE rollback.tbl AS SELECT range AS i FROM range(100);
statement ok
ROLLBACK;
statement error
ATTACH '__TEST_DIR__/rollback.db' (BLOCK_SIZE 262144);
----
block size parameter does not match

17
external/duckdb/test/sql/storage/compact_block_size/compact_block_size.test vendored Normal file
View File

@@ -0,0 +1,17 @@
# name: test/sql/storage/compact_block_size/compact_block_size.test
# description: Various tests with a compact block size of 16384 bytes
# group: [compact_block_size]
require exact_vector_size 2048
statement ok
ATTACH 'data/storage/index_0-9-1.db' (TYPE DUCKDB, READONLY);
# vector size is 2048, block size is 16KB
statement ok
ATTACH 'data/storage/block_size_16kb.db' (TYPE DUCKDB, READONLY);
query I
SELECT * FROM block_size_16kb.tbl;
----
42

19
external/duckdb/test/sql/storage/compact_block_size/compact_vector_size.test vendored Normal file
View File

@@ -0,0 +1,19 @@
# name: test/sql/storage/compact_block_size/compact_vector_size.test
# description: Various tests with a compact vector size of 512
# group: [compact_block_size]
require exact_vector_size 512
# The vector size of this file is 2048.
statement error
ATTACH 'data/storage/index_0-9-1.db' (TYPE DUCKDB, READONLY);
----
Cannot read database file
statement ok
ATTACH 'data/storage/vector_size_512.db' (TYPE DUCKDB, READONLY);
query I
SELECT * FROM vector_size_512.tbl;
----
42

188
external/duckdb/test/sql/storage/compact_block_size/create_table_compression.test vendored Normal file
View File

@@ -0,0 +1,188 @@
# name: test/sql/storage/compact_block_size/create_table_compression.test
# description: Test CREATE TABLE using compression options
# group: [compact_block_size]
# This test defaults to another compression function for larger block sizes,
# because the bitpacking groups fit the blocks.
require block_size 16384
statement ok
PRAGMA enable_verification
statement ok
CREATE TABLE T (a INTEGER USING COMPRESSION RLE)
statement ok
DROP TABLE T
statement error
CREATE TABLE T (a INTEGER USING COMPRESSION 'bla')
----
statement error
CREATE TABLE T (a INTEGER USING COMPRESSION )
----
Parser Error: syntax error at or near ")"
statement error
CREATE TABLE T (a INTEGER NOT NULL USING COMPRESSION )
----
Parser Error: syntax error at or near ")"
statement error
CREATE TABLE T (a INTEGER USING COMPRESSION bla)
----
statement ok
CREATE TABLE T (a INTEGER NOT NULL USING COMPRESSION RLE)
statement ok
DROP TABLE T
statement ok
CREATE TABLE T (a INTEGER USING COMPRESSION RLE, b VARCHAR )
statement ok
DROP TABLE T
load __TEST_DIR__/test_compression_hint.db
statement ok
CREATE TABLE T (a INTEGER USING COMPRESSION RLE, b INTEGER USING COMPRESSION BITPACKING, C INTEGER USING COMPRESSION UNCOMPRESSED)
statement ok
INSERT INTO T VALUES (1,1,1), (1,1,1), (1,1,1), (2,2,2), (2,2,2), (3,3,3)
query III
SELECT * FROM T
----
1 1 1
1 1 1
1 1 1
2 2 2
2 2 2
3 3 3
restart
query III
SELECT * FROM T
----
1 1 1
1 1 1
1 1 1
2 2 2
2 2 2
3 3 3
statement ok
CHECKPOINT
# we default to RLE instead of bitpacking
query I
SELECT COUNT(*) > 0 FROM pragma_storage_info('T')
WHERE segment_type ILIKE 'INTEGER' AND compression = 'RLE';
----
1
statement ok
ALTER TABLE T RENAME COLUMN a TO a_1
statement ok
ALTER TABLE T RENAME COLUMN b TO b_1
statement ok
ALTER TABLE T RENAME COLUMN c TO c_1
restart
query III
SELECT * FROM T
----
1 1 1
1 1 1
1 1 1
2 2 2
2 2 2
3 3 3
# we default to RLE instead of bitpacking
query I
SELECT COUNT(*) > 0 FROM pragma_storage_info('T')
WHERE segment_type ILIKE 'INTEGER' AND compression = 'RLE';
----
1
statement ok
ALTER TABLE T RENAME TO T_1
restart
query III
SELECT * FROM T_1
----
1 1 1
1 1 1
1 1 1
2 2 2
2 2 2
3 3 3
# we default to RLE instead of bitpacking
query I
SELECT COUNT(*) > 0 FROM pragma_storage_info('T_1')
WHERE segment_type ILIKE 'INTEGER' AND compression = 'RLE';
----
1
# Test Drop Column
statement ok
ALTER TABLE T_1 DROP COLUMN c_1
statement ok
ALTER TABLE T_1 DROP COLUMN b_1
restart
query I
SELECT * FROM T_1
----
1
1
1
2
2
3
query I
SELECT compression FROM pragma_storage_info('T_1') WHERE segment_type ILIKE 'INTEGER' LIMIT 2
----
RLE
# Test Add Column
statement ok
ALTER TABLE T_1 ADD COLUMN b INTEGER DEFAULT 2
restart
query II
SELECT * FROM T_1
----
1 2
1 2
1 2
2 2
2 2
3 2
statement ok
CHECKPOINT
query I
SELECT compression FROM pragma_storage_info('T_1') WHERE segment_type ILIKE 'INTEGER' LIMIT 3
----
RLE
Constant

20
external/duckdb/test/sql/storage/compact_block_size/default_block_size.test vendored Normal file
View File

@@ -0,0 +1,20 @@
# name: test/sql/storage/compact_block_size/default_block_size.test
# description: Various tests with a compact block size of 16384 bytes
# group: [compact_block_size]
# The database is written with a vector size of 2048.
require vector_size 2048
statement ok
ATTACH 'data/storage/block_size_16kb.db' (TYPE DUCKDB, READONLY)
statement error
ATTACH 'data/storage/vector_size_512.db' (TYPE DUCKDB, READONLY)
----
Cannot read database file
# Otherwise Linux 32bit CI fails with: No more data remaining in MetadataReader
require 64bit
statement ok
ATTACH 'data/storage/index_0-9-1.db' (TYPE DUCKDB, READONLY);

32
external/duckdb/test/sql/storage/compact_block_size/ensure_bitpacking.test vendored Normal file
View File

@@ -0,0 +1,32 @@
# name: test/sql/storage/compact_block_size/ensure_bitpacking.test
# description: Ensure that we serialize a bitpacking segment for 256KB databases.
# group: [compact_block_size]
statement ok
SET threads=1;
statement ok
PRAGMA wal_autocheckpoint='1TB';
statement ok
ATTACH '__TEST_DIR__/smaller_block_size.db' (BLOCK_SIZE 16384);
statement ok
CREATE TABLE smaller_block_size.tbl AS SELECT range AS i FROM range(10000);
statement ok
ATTACH '__TEST_DIR__/larger_block_size.db' (BLOCK_SIZE 262144);
statement ok
CREATE TABLE larger_block_size.tbl AS SELECT range AS i FROM range(10000);
statement ok
CHECKPOINT smaller_block_size;
statement ok
CHECKPOINT larger_block_size;
query I
SELECT COUNT(*) > 0 FROM pragma_storage_info('larger_block_size.tbl') WHERE compression = 'BitPacking';
----
1

32
external/duckdb/test/sql/storage/compact_block_size/ensure_no_bitpacking.test vendored Normal file
View File

@@ -0,0 +1,32 @@
# name: test/sql/storage/compact_block_size/ensure_no_bitpacking.test
# description: Ensure that we do not serialize a bitpacking segment for 16KB databases.
# group: [compact_block_size]
statement ok
SET threads=1;
statement ok
PRAGMA wal_autocheckpoint='1TB';
statement ok
ATTACH '__TEST_DIR__/no_bitpacking.db' (BLOCK_SIZE 16384);
statement ok
CREATE TABLE no_bitpacking.tbl AS SELECT range AS i FROM range(10000);
statement ok
ATTACH '__TEST_DIR__/has_bitpacking.db' (BLOCK_SIZE 262144);
statement ok
CREATE TABLE has_bitpacking.tbl AS SELECT range AS i FROM range(10000);
statement ok
CHECKPOINT has_bitpacking;
statement ok
CHECKPOINT no_bitpacking;
query I
SELECT COUNT(*) FROM pragma_storage_info('no_bitpacking.tbl') WHERE compression = 'BitPacking';
----
0

83
external/duckdb/test/sql/storage/compact_block_size/insertion_order_odd_batches.test vendored Normal file
View File

@@ -0,0 +1,83 @@
# name: test/sql/storage/compact_block_size/insertion_order_odd_batches.test
# description: Tests for insertion order preservation with compact blocks
# group: [compact_block_size]
# This test defaults to other counts for smaller block sizes.
require block_size 16384
require vector_size 512
require parquet
load __TEST_DIR__/compact_odd_batches.db
query I
CREATE TABLE integers AS SELECT * FROM range(100000) tbl(i);
----
100000
# check the block count and median number of rows per row group
query I
SELECT COUNT(DISTINCT block_id) < 60 FROM pragma_storage_info('integers');
----
true
query I
SELECT MEDIAN(count) FROM pragma_storage_info('integers');
----
2047
statement ok
COPY integers TO '__TEST_DIR__/integers.parquet' (ROW_GROUP_SIZE 777)
# verify that reading while preserving insertion order creates the same size table
statement ok
CREATE TABLE integers_parquet AS FROM '__TEST_DIR__/integers.parquet';
query I
SELECT * FROM integers_parquet LIMIT 5
----
0
1
2
3
4
query I
SELECT * FROM integers_parquet LIMIT 5 OFFSET 73654
----
73654
73655
73656
73657
73658
query I
SELECT COUNT(DISTINCT block_id) < 60 FROM pragma_storage_info('integers_parquet');
----
true
query I
SELECT MEDIAN(count) FROM pragma_storage_info('integers_parquet');
----
2047
# verify that reading without preserving insertion order creates the same size table
statement ok
SET preserve_insertion_order=false
statement ok
CREATE TABLE integers_parquet_no_order AS FROM '__TEST_DIR__/integers.parquet'
query I
SELECT COUNT(DISTINCT block_id) < 60 FROM pragma_storage_info('integers_parquet_no_order');
----
true
query I
SELECT MEDIAN(count) FROM pragma_storage_info('integers_parquet_no_order');
----
2047

21
external/duckdb/test/sql/storage/compact_block_size/mixed_block_sizes.test vendored Normal file
View File

@@ -0,0 +1,21 @@
# name: test/sql/storage/compact_block_size/mixed_block_sizes.test
# description: Tests queries with mixed block sizes.
# group: [compact_block_size]
statement ok
ATTACH '__TEST_DIR__/small.db' (BLOCK_SIZE 16384);
statement ok
CREATE TABLE small.tbl AS SELECT range AS i FROM range(10000);
statement ok
ATTACH '__TEST_DIR__/large.db' (BLOCK_SIZE 262144);
statement ok
CREATE TABLE large.tbl AS SELECT range AS i FROM range(10000);
query I
SELECT list_sum(LIST(t1.i) || LIST(t2.i))
FROM large.tbl AS t1 JOIN small.tbl AS t2 ON t1.i = t2.i;
----
99990000

335
external/duckdb/test/sql/storage/compression/alp/alp_inf_null_nan.test vendored Normal file
View File

@@ -0,0 +1,335 @@
# name: test/sql/storage/compression/alp/alp_inf_null_nan.test
# group: [alp]
# load the DB from disk
load __TEST_DIR__/test_alp_nulls.db
foreach compression uncompressed alp
# Set the compression algorithm
statement ok
pragma force_compression='${compression}'
# Create tables
statement ok
create table tbl1_${compression}(
a INTEGER DEFAULT 5,
b VARCHAR DEFAULT 'test',
c BOOL DEFAULT false,
d DOUBLE,
e TEXT default 'null',
f FLOAT
);
statement ok
create table tbl2_${compression}(
a INTEGER DEFAULT 5,
b VARCHAR DEFAULT 'test',
c BOOL DEFAULT false,
d DOUBLE,
e TEXT default 'null',
f FLOAT
);
statement ok
create table tbl3_${compression}(
a INTEGER DEFAULT 5,
b VARCHAR DEFAULT 'test',
c BOOL DEFAULT false,
d DOUBLE,
e TEXT default 'null',
f FLOAT
);
# Populate tables
# Mixed NULLs
statement ok
insert into tbl1_${compression}(d,f) VALUES
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity');
# Only NULLS
statement ok
insert into tbl2_${compression}(d,f) VALUES
(NULL, NULL),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', '-Infinity'),
('Infinity', '-Infinity'),
('Infinity', '-Infinity'),
('Infinity', '-Infinity'),
('Infinity', '-Infinity'),
('-Infinity', '-Infinity'),
('-Infinity', '-Infinity'),
('-Infinity', '-Infinity'),
('-Infinity', '-Infinity'),
('-Infinity', '-Infinity'),
('-Infinity', '-Infinity'),
(0, 0),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL);
# Starting with NULLS
statement ok
insert into tbl3_${compression}(d,f) VALUES
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', '-Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', '-Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', '-Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', '-Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', '-Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(NULL, NULL),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('NaN', 'NaN'),
('NaN', 'NaN'),
('NaN', 'NaN');
# Force a checkpoint
statement ok
checkpoint
endloop
# Assert that the scanned results are the same
#tbl1
query II nosort r1
select d, f from tbl1_uncompressed;
----
query II nosort r1
select d, f from tbl1_alp;
----
#tbl2
query II nosort r2
select d, f from tbl2_uncompressed;
----
query II nosort r2
select d, f from tbl2_alp;
----
# tbl3
query II nosort r3
select d, f from tbl3_uncompressed;
----
query II nosort r3
select d, f from tbl3_alp;
----

127
external/duckdb/test/sql/storage/compression/alp/alp_list_skip.test vendored Normal file
View File

@@ -0,0 +1,127 @@
# name: test/sql/storage/compression/alp/alp_list_skip.test
# description: Test skipping of small lists in alp
# group: [alp]
# load the DB from disk
load __TEST_DIR__/test_alp_list_skip.db
foreach comp alp alprd
statement ok
SET force_compression='${comp}'
# Create a table with random doubles of limited precision compressed as Uncompressed
# This data should achieve x6 compression ratio
statement ok
create or replace table list_doubles as select 5700 i, [5700.0] l UNION ALL select i, CASE WHEN i%128=0 THEN [i::DOUBLE] ELSE []::DOUBLE[] END as data from range(10000) tbl(i) union all select 5700, [i] FROM range(100) tbl(i);
statement ok
checkpoint
query II
SELECT * FROM list_doubles WHERE i=5700
----
5700 [5700.0]
5700 []
5700 [0.0]
5700 [1.0]
5700 [2.0]
5700 [3.0]
5700 [4.0]
5700 [5.0]
5700 [6.0]
5700 [7.0]
5700 [8.0]
5700 [9.0]
5700 [10.0]
5700 [11.0]
5700 [12.0]
5700 [13.0]
5700 [14.0]
5700 [15.0]
5700 [16.0]
5700 [17.0]
5700 [18.0]
5700 [19.0]
5700 [20.0]
5700 [21.0]
5700 [22.0]
5700 [23.0]
5700 [24.0]
5700 [25.0]
5700 [26.0]
5700 [27.0]
5700 [28.0]
5700 [29.0]
5700 [30.0]
5700 [31.0]
5700 [32.0]
5700 [33.0]
5700 [34.0]
5700 [35.0]
5700 [36.0]
5700 [37.0]
5700 [38.0]
5700 [39.0]
5700 [40.0]
5700 [41.0]
5700 [42.0]
5700 [43.0]
5700 [44.0]
5700 [45.0]
5700 [46.0]
5700 [47.0]
5700 [48.0]
5700 [49.0]
5700 [50.0]
5700 [51.0]
5700 [52.0]
5700 [53.0]
5700 [54.0]
5700 [55.0]
5700 [56.0]
5700 [57.0]
5700 [58.0]
5700 [59.0]
5700 [60.0]
5700 [61.0]
5700 [62.0]
5700 [63.0]
5700 [64.0]
5700 [65.0]
5700 [66.0]
5700 [67.0]
5700 [68.0]
5700 [69.0]
5700 [70.0]
5700 [71.0]
5700 [72.0]
5700 [73.0]
5700 [74.0]
5700 [75.0]
5700 [76.0]
5700 [77.0]
5700 [78.0]
5700 [79.0]
5700 [80.0]
5700 [81.0]
5700 [82.0]
5700 [83.0]
5700 [84.0]
5700 [85.0]
5700 [86.0]
5700 [87.0]
5700 [88.0]
5700 [89.0]
5700 [90.0]
5700 [91.0]
5700 [92.0]
5700 [93.0]
5700 [94.0]
5700 [95.0]
5700 [96.0]
5700 [97.0]
5700 [98.0]
5700 [99.0]
endloop

44
external/duckdb/test/sql/storage/compression/alp/alp_many_segments.test_slow vendored Normal file
View File

@@ -0,0 +1,44 @@
# name: test/sql/storage/compression/alp/alp_many_segments.test_slow
# description: Test storage of alp, but simple
# group: [alp]
# load the DB from disk
load __TEST_DIR__/test_alp.db
statement ok
PRAGMA force_compression='uncompressed'
# Create a table with random doubles of limited precision compressed as Uncompressed
# This data should achieve x6 compression ratio
statement ok
create table random_double as select round(random(), 6)::DOUBLE as data from range(500000) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_double') WHERE segment_type == 'double' AND compression != 'Uncompressed';
----
# Now create a duplicate of this table, compressed with ALP instead
statement ok
PRAGMA force_compression='alp'
statement ok
create table random_alp_double as select * from random_double;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_alp_double') WHERE segment_type == 'double' AND compression != 'ALP';
----
# Assert that the data was not corrupted by compressing to ALP
query I sort r1
select * from random_double;
----
query I sort r1
select * from random_alp_double;
----

44
external/duckdb/test/sql/storage/compression/alp/alp_many_segments_float.test_slow vendored Normal file
View File

@@ -0,0 +1,44 @@
# name: test/sql/storage/compression/alp/alp_many_segments_float.test_slow
# description: Test storage of alp, but simple
# group: [alp]
# load the DB from disk
load __TEST_DIR__/test_alp.db
statement ok
PRAGMA force_compression='uncompressed'
# Create a table with random floats of limited precision compressed as Uncompressed
# This data should achieve x6 compression ratio
statement ok
create table random_float as select round(random(), 3)::FLOAT as data from range(500000) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_float') WHERE segment_type == 'float' AND compression != 'Uncompressed';
----
# Now create a duplicate of this table, compressed with ALP instead
statement ok
PRAGMA force_compression='alp'
statement ok
create table random_alp_float as select * from random_float;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_alp_float') WHERE segment_type == 'float' AND compression != 'ALP';
----
# Assert that the data was not corrupted by compressing to ALP
query I sort r1
select * from random_float;
----
query I sort r1
select * from random_alp_float;
----

44
external/duckdb/test/sql/storage/compression/alp/alp_middle_flush.test_slow vendored Normal file
View File

@@ -0,0 +1,44 @@
# name: test/sql/storage/compression/alp/alp_middle_flush.test_slow
# description: Test storage of alp, but simple
# group: [alp]
# load the DB from disk
load __TEST_DIR__/test_alp.db
statement ok
PRAGMA force_compression='uncompressed'
# Create a table with random doubles of limited precision compressed as Uncompressed
# This data should achieve x6 compression ratio
statement ok
create table random_double as select round(random(), 6)::DOUBLE as data from range(110000) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_double') WHERE segment_type == 'double' AND compression != 'Uncompressed';
----
# Now create a duplicate of this table, compressed with ALP instead
statement ok
PRAGMA force_compression='alp'
statement ok
create table random_alp_double as select * from random_double;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_alp_double') WHERE segment_type == 'double' AND compression != 'ALP';
----
# Assert that the data was not corrupted by compressing to ALP
query I sort r1
select * from random_double;
----
query I sort r1
select * from random_alp_double;
----

36
external/duckdb/test/sql/storage/compression/alp/alp_min_max.test vendored Normal file
View File

@@ -0,0 +1,36 @@
# name: test/sql/storage/compression/alp/alp_min_max.test
# group: [alp]
load __TEST_DIR__/alp_min_max.db
statement ok
PRAGMA enable_verification
statement ok
PRAGMA force_compression='alp';
foreach type DOUBLE FLOAT
statement ok
CREATE TABLE all_types AS SELECT ${type} FROM test_all_types();
loop i 0 15
statement ok
INSERT INTO all_types SELECT ${type} FROM all_types;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('all_types') WHERE segment_type == '${type}' AND compression != 'ALP';
----
# i
endloop
statement ok
DROP TABLE all_types;
#type
endloop

45
external/duckdb/test/sql/storage/compression/alp/alp_negative_numbers.test vendored Normal file
View File

@@ -0,0 +1,45 @@
# name: test/sql/storage/compression/alp/alp_negative_numbers.test
# description: Test storage of alp, but simple
# group: [alp]
# load the DB from disk
load __TEST_DIR__/test_alp.db
statement ok
PRAGMA force_compression='uncompressed'
# Create a table with random doubles of limited precision compressed as Uncompressed
# This data should achieve x6 compression ratio
statement ok
create table random_double as select round(cos(1 / (random() + 0.001)), 5)::DOUBLE * -1 as data from range(1023) tbl(i);
insert into random_double values (-0.0::DOUBLE);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_double') WHERE segment_type == 'double' AND compression != 'Uncompressed';
----
# Now create a duplicate of this table, compressed with ALP instead
statement ok
PRAGMA force_compression='alp'
statement ok
create table random_alp_double as select * from random_double;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_alp_double') WHERE segment_type == 'double' AND compression != 'ALP';
----
# Assert that the data was not corrupted by compressing to ALP
query I sort r1
select * from random_double;
----
query I sort r1
select * from random_alp_double;
----

342
external/duckdb/test/sql/storage/compression/alp/alp_nulls.test vendored Normal file
View File

@@ -0,0 +1,342 @@
# name: test/sql/storage/compression/alp/alp_nulls.test
# group: [alp]
# load the DB from disk
load __TEST_DIR__/test_alp_nulls.db
foreach compression uncompressed alp
# Set the compression algorithm
statement ok
pragma force_compression='${compression}'
# Create tables
statement ok
create table tbl1_${compression}(
a INTEGER DEFAULT 5,
b VARCHAR DEFAULT 'test',
c BOOL DEFAULT false,
d DOUBLE,
e TEXT default 'null',
f FLOAT
);
statement ok
create table tbl2_${compression}(
a INTEGER DEFAULT 5,
b VARCHAR DEFAULT 'test',
c BOOL DEFAULT false,
d DOUBLE,
e TEXT default 'null',
f FLOAT
);
statement ok
create table tbl3_${compression}(
a INTEGER DEFAULT 5,
b VARCHAR DEFAULT 'test',
c BOOL DEFAULT false,
d DOUBLE,
e TEXT default 'null',
f FLOAT
);
# Populate tables
# Mixed NULLs
statement ok
insert into tbl1_${compression}(d,f) VALUES
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, NULL);
# Only NULLS
statement ok
insert into tbl2_${compression}(d,f) VALUES
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL);
# Starting with NULLS
statement ok
insert into tbl3_${compression}(d,f) VALUES
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142);
# Force a checkpoint
statement ok
checkpoint
endloop
# Assert that the scanned results are the same
#tbl1
query II nosort r1
select d, f from tbl1_uncompressed;
----
query II nosort r1
select d, f from tbl1_alp;
----
#tbl2
query II nosort r2
select d, f from tbl2_uncompressed;
----
query II nosort r2
select d, f from tbl2_alp;
----
# tbl3
query II nosort r3
select d, f from tbl3_uncompressed;
----
query II nosort r3
select d, f from tbl3_alp;
----

129
external/duckdb/test/sql/storage/compression/alp/alp_nulls_simple.test vendored Normal file
View File

@@ -0,0 +1,129 @@
# name: test/sql/storage/compression/alp/alp_nulls_simple.test
# group: [alp]
# load the DB from disk
load __TEST_DIR__/test_alp_nulls.db
foreach compression uncompressed alp
# Set the compression algorithm
statement ok
pragma force_compression='${compression}'
# Create tables
statement ok
create table tbl1_${compression}(
a INTEGER DEFAULT 5,
b VARCHAR DEFAULT 'test',
c BOOL DEFAULT false,
d DOUBLE,
e TEXT default 'null',
f FLOAT
);
# Populate tables
# Mixed NULLs
statement ok
insert into tbl1_${compression}(d,f) VALUES
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, NULL);
# Force a checkpoint
statement ok
checkpoint
endloop
# Assert that the scanned results are the same
#tbl1
query II nosort r1
select d, f from tbl1_uncompressed;
----
query II nosort r1
select d, f from tbl1_alp;
----

51
external/duckdb/test/sql/storage/compression/alp/alp_read.test_slow vendored Normal file
View File

@@ -0,0 +1,51 @@
# name: test/sql/storage/compression/alp/alp_read.test_slow
# group: [alp]
require parquet
require httpfs
load __TEST_DIR__/test_alp.db
statement ok
DROP TABLE IF EXISTS temperatures;
statement ok
pragma threads=1;
statement ok
PRAGMA force_compression='alp';
statement ok
CREATE TABLE temperatures (
temperature DOUBLE
);
statement ok
INSERT INTO temperatures SELECT temp from 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/city_temperature.parquet' t(temp), range(1);
statement ok
CHECKPOINT;
statement ok
create table reference_temperatures (
temperature DOUBLE
);
statement ok
PRAGMA force_compression='uncompressed';
statement ok
INSERT INTO reference_temperatures SELECT temp from 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/city_temperature.parquet' t(temp), range(1);
statement ok
checkpoint;
query I nosort r1
select temperature from reference_temperatures;
----
query I nosort r1
select temperature from temperatures
----

44
external/duckdb/test/sql/storage/compression/alp/alp_simple.test vendored Normal file
View File

@@ -0,0 +1,44 @@
# name: test/sql/storage/compression/alp/alp_simple.test
# description: Test storage of alp, but simple
# group: [alp]
# load the DB from disk
load __TEST_DIR__/test_alp.db
statement ok
PRAGMA force_compression='uncompressed'
# Create a table with random doubles of limited precision compressed as Uncompressed
# This data should achieve x6 compression ratio
statement ok
create table random_double as select round(random(), 6)::DOUBLE as data from range(1024) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_double') WHERE segment_type == 'double' AND compression != 'Uncompressed';
----
# Now create a duplicate of this table, compressed with ALP instead
statement ok
PRAGMA force_compression='alp'
statement ok
create table random_alp_double as select * from random_double;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_alp_double') WHERE segment_type == 'double' AND compression != 'ALP';
----
# Assert that the data was not corrupted by compressing to ALP
query I sort r1
select * from random_double;
----
query I sort r1
select * from random_alp_double;
----

44
external/duckdb/test/sql/storage/compression/alp/alp_simple_float.test vendored Normal file
View File

@@ -0,0 +1,44 @@
# name: test/sql/storage/compression/alp/alp_simple_float.test
# description: Test storage of alp, but simple
# group: [alp]
# load the DB from disk
load __TEST_DIR__/test_alp.db
statement ok
PRAGMA force_compression='uncompressed'
# Create a table with random floats of limited precision compressed as Uncompressed
# This data should achieve x6 compression ratio
statement ok
create table random_float as select round(random(), 6)::FLOAT as data from range(1024) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_float') WHERE segment_type == 'float' AND compression != 'Uncompressed';
----
# Now create a duplicate of this table, compressed with ALP instead
statement ok
PRAGMA force_compression='alp'
statement ok
create table random_alp_float as select * from random_float;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_alp_float') WHERE segment_type == 'float' AND compression != 'ALP';
----
# Assert that the data was not corrupted by compressing to ALP
query I sort r1
select * from random_float;
----
query I sort r1
select * from random_alp_float;
----

55
external/duckdb/test/sql/storage/compression/alp/alp_skip.test_slow vendored Normal file
View File

@@ -0,0 +1,55 @@
# name: test/sql/storage/compression/alp/alp_skip.test_slow
# group: [alp]
# load the DB from disk
load __TEST_DIR__/test_alp.db
statement ok
pragma enable_verification;
statement ok
pragma disable_optimizer;
statement ok
pragma force_compression='uncompressed'
# Create the data for the columns
statement ok
create table temp_table as select round(random(), 6)::DOUBLE * 100 as col, j from range(10240) tbl(j);
statement ok
checkpoint
foreach compression ALP Uncompressed
# Ensure the correct compression is used
statement ok
pragma force_compression='${compression}'
# Setup
statement ok
create table tbl_${compression} as select * from temp_table;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('tbl_${compression}') WHERE segment_type == 'double' AND compression != '${compression}';
----
# compression
endloop
loop i 1 1024
query II
select x as x_${i}, y as y_${i} from (
select
(select col from tbl_alp where (j > (${i} * 1024)) except select col from tbl_uncompressed where (j > (${i} * 1024))) as x,
(select col from tbl_uncompressed where (j > (${i} * 1024)) except select col from tbl_alp where (j > (${i} * 1024))) as y
);
----
NULL NULL
# i
endloop

94
external/duckdb/test/sql/storage/compression/alp/alp_stress_test.test_slow vendored Normal file
View File

@@ -0,0 +1,94 @@
# name: test/sql/storage/compression/alp/alp_stress_test.test_slow
# group: [alp]
load __TEST_DIR__/alp_min_max.db
foreach type DOUBLE FLOAT
statement ok
pragma force_compression='uncompressed'
# Create the initial data to start with (without this Constant compression will be used at first)
statement ok
create table temp_table as select round(random(), 6)::${type} * 100 from range(5);
statement ok
checkpoint
foreach compression ALP Uncompressed
# Ensure the correct compression is used
statement ok
pragma force_compression='${compression}'
# Setup
statement ok
create table ${compression}_tbl(
data ${type}
);
statement ok
insert into ${compression}_tbl select * from temp_table;
statement ok
checkpoint
# compression
endloop
statement ok
drop table temp_table;
# ---------------- MAIN LOOP ----------------
loop i 0 10240
# Create a temporary table containing the value we want to add to both tables
statement ok
create table temp_table as select random()::${type} * 100 + ${i};
foreach compression ALP Uncompressed
# Ensure the correct compression is used
statement ok
pragma force_compression='${compression}'
# Insert a single value into the table
statement ok
insert into ${compression}_tbl select * from temp_table;
# Checkpoint the table with the newly added data
statement ok
checkpoint
# compression
endloop
statement ok
drop table temp_table;
# ---------------- VERIFY CORRECTNESS ----------------
query II
select x as x_${i}, y as y_${i} from (
select
(select data from alp_tbl except select data from uncompressed_tbl) as x,
(select data from uncompressed_tbl except select data from alp_tbl) as y
);
----
NULL NULL
# i
endloop
# Teardown
foreach compression alp uncompressed
statement ok
drop table ${compression}_tbl;
# compression
endloop
# type
endloop

753
external/duckdb/test/sql/storage/compression/alp/alp_tpcds.test_slow vendored Normal file
View File

@@ -0,0 +1,753 @@
# name: test/sql/storage/compression/alp/alp_tpcds.test_slow
# group: [alp]
require tpcds
# load the DB from disk
load __TEST_DIR__/test_alp.db
#statement ok
#pragma threads=1
statement ok
PRAGMA force_compression='alp';
statement ok
call dsdgen(sf=1, suffix='_original');
statement ok
PRAGMA default_null_order='NULLS LAST'
# Test both DOUBLE and FLOAT
foreach type DOUBLE FLOAT
# Create tables
statement ok
CREATE TABLE web_site(
web_site_sk ${type},
web_site_id VARCHAR,
web_rec_start_date DATE,
web_rec_end_date DATE,
web_name VARCHAR,
web_open_date_sk ${type},
web_close_date_sk ${type},
web_class VARCHAR,
web_manager VARCHAR,
web_mkt_id ${type},
web_mkt_class VARCHAR,
web_mkt_desc VARCHAR,
web_market_manager VARCHAR,
web_company_id ${type},
web_company_name VARCHAR,
web_street_number VARCHAR,
web_street_name VARCHAR,
web_street_type VARCHAR,
web_suite_number VARCHAR,
web_city VARCHAR,
web_county VARCHAR,
web_state VARCHAR,
web_zip VARCHAR,
web_country VARCHAR,
web_gmt_offset ${type},
web_tax_percentage ${type}
);
statement ok
CREATE TABLE web_sales(
ws_sold_date_sk ${type},
ws_sold_time_sk ${type},
ws_ship_date_sk ${type},
ws_item_sk ${type},
ws_bill_customer_sk ${type},
ws_bill_cdemo_sk ${type},
ws_bill_hdemo_sk ${type},
ws_bill_addr_sk ${type},
ws_ship_customer_sk ${type},
ws_ship_cdemo_sk ${type},
ws_ship_hdemo_sk ${type},
ws_ship_addr_sk ${type},
ws_web_page_sk ${type},
ws_web_site_sk ${type},
ws_ship_mode_sk ${type},
ws_warehouse_sk ${type},
ws_promo_sk ${type},
ws_order_number ${type},
ws_quantity ${type},
ws_wholesale_cost ${type},
ws_list_price ${type},
ws_sales_price ${type},
ws_ext_discount_amt ${type},
ws_ext_sales_price ${type},
ws_ext_wholesale_cost ${type},
ws_ext_list_price ${type},
ws_ext_tax ${type},
ws_coupon_amt ${type},
ws_ext_ship_cost ${type},
ws_net_paid ${type},
ws_net_paid_inc_tax ${type},
ws_net_paid_inc_ship ${type},
ws_net_paid_inc_ship_tax ${type},
ws_net_profit ${type}
);
statement ok
CREATE TABLE web_returns(
wr_returned_date_sk ${type},
wr_returned_time_sk ${type},
wr_item_sk ${type},
wr_refunded_customer_sk ${type},
wr_refunded_cdemo_sk ${type},
wr_refunded_hdemo_sk ${type},
wr_refunded_addr_sk ${type},
wr_returning_customer_sk ${type},
wr_returning_cdemo_sk ${type},
wr_returning_hdemo_sk ${type},
wr_returning_addr_sk ${type},
wr_web_page_sk ${type},
wr_reason_sk ${type},
wr_order_number ${type},
wr_return_quantity ${type},
wr_return_amt ${type},
wr_return_tax ${type},
wr_return_amt_inc_tax ${type},
wr_fee ${type},
wr_return_ship_cost ${type},
wr_refunded_cash ${type},
wr_reversed_charge ${type},
wr_account_credit ${type},
wr_net_loss ${type}
);
statement ok
CREATE TABLE web_page(
wp_web_page_sk ${type},
wp_web_page_id VARCHAR,
wp_rec_start_date DATE,
wp_rec_end_date DATE,
wp_creation_date_sk ${type},
wp_access_date_sk ${type},
wp_autogen_flag VARCHAR,
wp_customer_sk ${type},
wp_url VARCHAR,
wp_type VARCHAR,
wp_char_count ${type},
wp_link_count ${type},
wp_image_count ${type},
wp_max_ad_count ${type}
);
statement ok
CREATE TABLE warehouse(
w_warehouse_sk ${type},
w_warehouse_id VARCHAR,
w_warehouse_name VARCHAR,
w_warehouse_sq_ft ${type},
w_street_number VARCHAR,
w_street_name VARCHAR,
w_street_type VARCHAR,
w_suite_number VARCHAR,
w_city VARCHAR,
w_county VARCHAR,
w_state VARCHAR,
w_zip VARCHAR,
w_country VARCHAR,
w_gmt_offset ${type}
);
statement ok
CREATE TABLE time_dim(
t_time_sk ${type},
t_time_id VARCHAR,
t_time ${type},
t_hour ${type},
t_minute ${type},
t_second ${type},
t_am_pm VARCHAR,
t_shift VARCHAR,
t_sub_shift VARCHAR,
t_meal_time VARCHAR
);
statement ok
CREATE TABLE store_sales(
ss_sold_date_sk ${type},
ss_sold_time_sk ${type},
ss_item_sk ${type},
ss_customer_sk ${type},
ss_cdemo_sk ${type},
ss_hdemo_sk ${type},
ss_addr_sk ${type},
ss_store_sk ${type},
ss_promo_sk ${type},
ss_ticket_number ${type},
ss_quantity ${type},
ss_wholesale_cost ${type},
ss_list_price ${type},
ss_sales_price ${type},
ss_ext_discount_amt ${type},
ss_ext_sales_price ${type},
ss_ext_wholesale_cost ${type},
ss_ext_list_price ${type},
ss_ext_tax ${type},
ss_coupon_amt ${type},
ss_net_paid ${type},
ss_net_paid_inc_tax ${type},
ss_net_profit ${type}
);
statement ok
CREATE TABLE store_returns(
sr_returned_date_sk ${type},
sr_return_time_sk ${type},
sr_item_sk ${type},
sr_customer_sk ${type},
sr_cdemo_sk ${type},
sr_hdemo_sk ${type},
sr_addr_sk ${type},
sr_store_sk ${type},
sr_reason_sk ${type},
sr_ticket_number ${type},
sr_return_quantity ${type},
sr_return_amt ${type},
sr_return_tax ${type},
sr_return_amt_inc_tax ${type},
sr_fee ${type},
sr_return_ship_cost ${type},
sr_refunded_cash ${type},
sr_reversed_charge ${type},
sr_store_credit ${type},
sr_net_loss ${type}
);
statement ok
CREATE TABLE store(
s_store_sk ${type},
s_store_id VARCHAR,
s_rec_start_date DATE,
s_rec_end_date DATE,
s_closed_date_sk ${type},
s_store_name VARCHAR,
s_number_employees ${type},
s_floor_space ${type},
s_hours VARCHAR,
s_manager VARCHAR,
s_market_id ${type},
s_geography_class VARCHAR,
s_market_desc VARCHAR,
s_market_manager VARCHAR,
s_division_id ${type},
s_division_name VARCHAR,
s_company_id ${type},
s_company_name VARCHAR,
s_street_number VARCHAR,
s_street_name VARCHAR,
s_street_type VARCHAR,
s_suite_number VARCHAR,
s_city VARCHAR,
s_county VARCHAR,
s_state VARCHAR,
s_zip VARCHAR,
s_country VARCHAR,
s_gmt_offset ${type},
s_tax_percentage ${type}
);
statement ok
CREATE TABLE ship_mode(
sm_ship_mode_sk ${type},
sm_ship_mode_id VARCHAR,
sm_type VARCHAR,
sm_code VARCHAR,
sm_carrier VARCHAR,
sm_contract VARCHAR
);
statement ok
CREATE TABLE reason(
r_reason_sk ${type},
r_reason_id VARCHAR,
r_reason_desc VARCHAR
);
statement ok
CREATE TABLE promotion(
p_promo_sk ${type},
p_promo_id VARCHAR,
p_start_date_sk ${type},
p_end_date_sk ${type},
p_item_sk ${type},
p_cost ${type},
p_response_target ${type},
p_promo_name VARCHAR,
p_channel_dmail VARCHAR,
p_channel_email VARCHAR,
p_channel_catalog VARCHAR,
p_channel_tv VARCHAR,
p_channel_radio VARCHAR,
p_channel_press VARCHAR,
p_channel_event VARCHAR,
p_channel_demo VARCHAR,
p_channel_details VARCHAR,
p_purpose VARCHAR,
p_discount_active VARCHAR
);
statement ok
CREATE TABLE item(
i_item_sk ${type},
i_item_id VARCHAR,
i_rec_start_date DATE,
i_rec_end_date DATE,
i_item_desc VARCHAR,
i_current_price ${type},
i_wholesale_cost ${type},
i_brand_id ${type},
i_brand VARCHAR,
i_class_id ${type},
i_class VARCHAR,
i_category_id ${type},
i_category VARCHAR,
i_manufact_id ${type},
i_manufact VARCHAR,
i_size VARCHAR,
i_formulation VARCHAR,
i_color VARCHAR,
i_units VARCHAR,
i_container VARCHAR,
i_manager_id ${type},
i_product_name VARCHAR
);
statement ok
CREATE TABLE inventory(
inv_date_sk ${type},
inv_item_sk ${type},
inv_warehouse_sk ${type},
inv_quantity_on_hand ${type}
);
statement ok
CREATE TABLE income_band(
ib_income_band_sk ${type},
ib_lower_bound ${type},
ib_upper_bound ${type}
);
statement ok
CREATE TABLE household_demographics(
hd_demo_sk ${type},
hd_income_band_sk ${type},
hd_buy_potential VARCHAR,
hd_dep_count ${type},
hd_vehicle_count ${type}
);
statement ok
CREATE TABLE date_dim(
d_date_sk ${type},
d_date_id VARCHAR,
d_date DATE,
d_month_seq ${type},
d_week_seq ${type},
d_quarter_seq ${type},
d_year ${type},
d_dow ${type},
d_moy ${type},
d_dom ${type},
d_qoy ${type},
d_fy_year ${type},
d_fy_quarter_seq ${type},
d_fy_week_seq ${type},
d_day_name VARCHAR,
d_quarter_name VARCHAR,
d_holiday VARCHAR,
d_weekend VARCHAR,
d_following_holiday VARCHAR,
d_first_dom ${type},
d_last_dom ${type},
d_same_day_ly ${type},
d_same_day_lq ${type},
d_current_day VARCHAR,
d_current_week VARCHAR,
d_current_month VARCHAR,
d_current_quarter VARCHAR,
d_current_year VARCHAR
);
statement ok
CREATE TABLE customer_demographics(
cd_demo_sk ${type},
cd_gender VARCHAR,
cd_marital_status VARCHAR,
cd_education_status VARCHAR,
cd_purchase_estimate ${type},
cd_credit_rating VARCHAR,
cd_dep_count ${type},
cd_dep_employed_count ${type},
cd_dep_college_count ${type}
);
statement ok
CREATE TABLE customer_address(
ca_address_sk ${type},
ca_address_id VARCHAR,
ca_street_number VARCHAR,
ca_street_name VARCHAR,
ca_street_type VARCHAR,
ca_suite_number VARCHAR,
ca_city VARCHAR,
ca_county VARCHAR,
ca_state VARCHAR,
ca_zip VARCHAR,
ca_country VARCHAR,
ca_gmt_offset ${type},
ca_location_type VARCHAR
);
statement ok
CREATE TABLE customer(
c_customer_sk ${type},
c_customer_id VARCHAR,
c_current_cdemo_sk ${type},
c_current_hdemo_sk ${type},
c_current_addr_sk ${type},
c_first_shipto_date_sk ${type},
c_first_sales_date_sk ${type},
c_salutation VARCHAR,
c_first_name VARCHAR,
c_last_name VARCHAR,
c_preferred_cust_flag VARCHAR,
c_birth_day ${type},
c_birth_month ${type},
c_birth_year ${type},
c_birth_country VARCHAR,
c_login VARCHAR,
c_email_address VARCHAR,
c_last_review_date_sk ${type}
);
statement ok
CREATE TABLE catalog_sales(
cs_sold_date_sk ${type},
cs_sold_time_sk ${type},
cs_ship_date_sk ${type},
cs_bill_customer_sk ${type},
cs_bill_cdemo_sk ${type},
cs_bill_hdemo_sk ${type},
cs_bill_addr_sk ${type},
cs_ship_customer_sk ${type},
cs_ship_cdemo_sk ${type},
cs_ship_hdemo_sk ${type},
cs_ship_addr_sk ${type},
cs_call_center_sk ${type},
cs_catalog_page_sk ${type},
cs_ship_mode_sk ${type},
cs_warehouse_sk ${type},
cs_item_sk ${type},
cs_promo_sk ${type},
cs_order_number ${type},
cs_quantity ${type},
cs_wholesale_cost ${type},
cs_list_price ${type},
cs_sales_price ${type},
cs_ext_discount_amt ${type},
cs_ext_sales_price ${type},
cs_ext_wholesale_cost ${type},
cs_ext_list_price ${type},
cs_ext_tax ${type},
cs_coupon_amt ${type},
cs_ext_ship_cost ${type},
cs_net_paid ${type},
cs_net_paid_inc_tax ${type},
cs_net_paid_inc_ship ${type},
cs_net_paid_inc_ship_tax ${type},
cs_net_profit ${type}
);
statement ok
CREATE TABLE catalog_returns(
cr_returned_date_sk ${type},
cr_returned_time_sk ${type},
cr_item_sk ${type},
cr_refunded_customer_sk ${type},
cr_refunded_cdemo_sk ${type},
cr_refunded_hdemo_sk ${type},
cr_refunded_addr_sk ${type},
cr_returning_customer_sk ${type},
cr_returning_cdemo_sk ${type},
cr_returning_hdemo_sk ${type},
cr_returning_addr_sk ${type},
cr_call_center_sk ${type},
cr_catalog_page_sk ${type},
cr_ship_mode_sk ${type},
cr_warehouse_sk ${type},
cr_reason_sk ${type},
cr_order_number ${type},
cr_return_quantity ${type},
cr_return_amount ${type},
cr_return_tax ${type},
cr_return_amt_inc_tax ${type},
cr_fee ${type},
cr_return_ship_cost ${type},
cr_refunded_cash ${type},
cr_reversed_charge ${type},
cr_store_credit ${type},
cr_net_loss ${type}
);
statement ok
CREATE TABLE catalog_page(
cp_catalog_page_sk ${type},
cp_catalog_page_id VARCHAR,
cp_start_date_sk ${type},
cp_end_date_sk ${type},
cp_department VARCHAR,
cp_catalog_number ${type},
cp_catalog_page_number ${type},
cp_description VARCHAR,
cp_type VARCHAR
);
statement ok
CREATE TABLE call_center(
cc_call_center_sk ${type},
cc_call_center_id VARCHAR,
cc_rec_start_date DATE,
cc_rec_end_date DATE,
cc_closed_date_sk ${type},
cc_open_date_sk ${type},
cc_name VARCHAR,
cc_class VARCHAR,
cc_employees ${type},
cc_sq_ft ${type},
cc_hours VARCHAR,
cc_manager VARCHAR,
cc_mkt_id ${type},
cc_mkt_class VARCHAR,
cc_mkt_desc VARCHAR,
cc_market_manager VARCHAR,
cc_division ${type},
cc_division_name VARCHAR,
cc_company ${type},
cc_company_name VARCHAR,
cc_street_number VARCHAR,
cc_street_name VARCHAR,
cc_street_type VARCHAR,
cc_suite_number VARCHAR,
cc_city VARCHAR,
cc_county VARCHAR,
cc_state VARCHAR,
cc_zip VARCHAR,
cc_country VARCHAR,
cc_gmt_offset ${type},
cc_tax_percentage ${type}
);
# Populate tables
statement ok
insert into web_site select * from web_site_original;
statement ok
insert into web_sales select * from web_sales_original;
statement ok
insert into web_returns select * from web_returns_original;
statement ok
insert into web_page select * from web_page_original;
statement ok
insert into warehouse select * from warehouse_original;
statement ok
insert into time_dim select * from time_dim_original;
statement ok
insert into store_sales select * from store_sales_original;
statement ok
insert into store_returns select * from store_returns_original;
statement ok
insert into store select * from store_original;
statement ok
insert into ship_mode select * from ship_mode_original;
statement ok
insert into reason select * from reason_original;
statement ok
insert into promotion select * from promotion_original;
statement ok
insert into item select * from item_original;
statement ok
insert into inventory select * from inventory_original;
statement ok
insert into income_band select * from income_band_original;
statement ok
insert into household_demographics select * from household_demographics_original;
statement ok
insert into date_dim select * from date_dim_original;
statement ok
insert into customer_demographics select * from customer_demographics_original;
statement ok
insert into customer_address select * from customer_address_original;
statement ok
insert into customer select * from customer_original;
statement ok
insert into catalog_sales select * from catalog_sales_original;
statement ok
insert into catalog_returns select * from catalog_returns_original;
statement ok
insert into catalog_page select * from catalog_page_original;
statement ok
insert into call_center select * from call_center_original;
# Checkpoint to compress the data
statement ok
checkpoint
# And verify that no other compression is used
foreach tbl web_site web_sales web_returns web_page warehouse time_dim store_sales store_returns store ship_mode reason promotion item inventory income_band household_demographics date_dim customer_demographics customer_address customer catalog_sales catalog_returns catalog_page call_center
# Cant turn off the creation of constant segments, so we have to just accept that some of the segments are Constant
query I
SELECT compression FROM pragma_storage_info('${tbl}') WHERE segment_type == '${type}' AND compression != 'ALP' AND compression != 'Constant';
----
endloop
# Run the tpcds queries
loop i 1 9
query I
PRAGMA tpcds(${i})
----
<FILE>:extension/tpcds/dsdgen/answers/sf1/0${i}.csv
endloop
loop i 10 49
#Skip tpcds 49 because it doesn't work without decimals
query I
PRAGMA tpcds(${i})
----
<FILE>:extension/tpcds/dsdgen/answers/sf1/${i}.csv
endloop
# skip tpcds 67 - inconsistent without decimals
loop i 50 66
query I
PRAGMA tpcds(${i})
----
<FILE>:extension/tpcds/dsdgen/answers/sf1/${i}.csv
endloop
loop i 68 99
query I
PRAGMA tpcds(${i})
----
<FILE>:extension/tpcds/dsdgen/answers/sf1/${i}.csv
endloop
# Drop tables
statement ok
DROP TABLE web_site;
statement ok
DROP TABLE web_sales;
statement ok
DROP TABLE web_returns;
statement ok
DROP TABLE web_page;
statement ok
DROP TABLE warehouse;
statement ok
DROP TABLE time_dim;
statement ok
DROP TABLE store_sales;
statement ok
DROP TABLE store_returns;
statement ok
DROP TABLE store;
statement ok
DROP TABLE ship_mode;
statement ok
DROP TABLE reason;
statement ok
DROP TABLE promotion;
statement ok
DROP TABLE item;
statement ok
DROP TABLE inventory;
statement ok
DROP TABLE income_band;
statement ok
DROP TABLE household_demographics;
statement ok
DROP TABLE date_dim;
statement ok
DROP TABLE customer_demographics;
statement ok
DROP TABLE customer_address;
statement ok
DROP TABLE customer;
statement ok
DROP TABLE catalog_sales;
statement ok
DROP TABLE catalog_returns;
statement ok
DROP TABLE catalog_page;
statement ok
DROP TABLE call_center;
endloop

247
external/duckdb/test/sql/storage/compression/alp/alp_tpch.test_slow vendored Normal file
View File

@@ -0,0 +1,247 @@
# name: test/sql/storage/compression/alp/alp_tpch.test_slow
# group: [alp]
require tpch
# load the DB from disk
load __TEST_DIR__/test_alp.db
# This needs to be single-threaded to be consistent (because of floating point issues)
statement ok
pragma threads=1
statement ok
PRAGMA force_compression='alp';
statement ok
call dbgen(sf=1, suffix='_original');
# Test both DOUBLE and FLOAT
foreach type DOUBLE FLOAT
# Create tables
statement ok
CREATE TABLE lineitem(
l_orderkey ${type} NOT NULL,
l_partkey ${type} NOT NULL,
l_suppkey ${type} NOT NULL,
l_linenumber ${type} NOT NULL,
l_quantity ${type} NOT NULL,
l_extendedprice ${type} NOT NULL,
l_discount ${type} NOT NULL,
l_tax ${type} NOT NULL,
l_returnflag VARCHAR NOT NULL,
l_linestatus VARCHAR NOT NULL,
l_shipdate DATE NOT NULL,
l_commitdate DATE NOT NULL,
l_receiptdate DATE NOT NULL,
l_shipinstruct VARCHAR NOT NULL,
l_shipmode VARCHAR NOT NULL,
l_comment VARCHAR NOT NULL
);
statement ok
CREATE TABLE orders(
o_orderkey ${type} NOT NULL,
o_custkey ${type} NOT NULL,
o_orderstatus VARCHAR NOT NULL,
o_totalprice ${type} NOT NULL,
o_orderdate DATE NOT NULL,
o_orderpriority VARCHAR NOT NULL,
o_clerk VARCHAR NOT NULL,
o_shippriority ${type} NOT NULL,
o_comment VARCHAR NOT NULL
);
statement ok
CREATE TABLE partsupp(
ps_partkey ${type} NOT NULL,
ps_suppkey ${type} NOT NULL,
ps_availqty ${type} NOT NULL,
ps_supplycost ${type} NOT NULL,
ps_comment VARCHAR NOT NULL
);
# 'p_partkey' being INTEGER is imperative to TPCH(17)
statement ok
CREATE TABLE part(
p_partkey INTEGER NOT NULL,
p_name VARCHAR NOT NULL,
p_mfgr VARCHAR NOT NULL,
p_brand VARCHAR NOT NULL,
p_type VARCHAR NOT NULL,
p_size ${type} NOT NULL,
p_container VARCHAR NOT NULL,
p_retailprice ${type} NOT NULL,
p_comment VARCHAR NOT NULL
);
statement ok
CREATE TABLE customer(
c_custkey ${type} NOT NULL,
c_name VARCHAR NOT NULL,
c_address VARCHAR NOT NULL,
c_nationkey ${type} NOT NULL,
c_phone VARCHAR NOT NULL,
c_acctbal ${type} NOT NULL,
c_mktsegment VARCHAR NOT NULL,
c_comment VARCHAR NOT NULL
);
statement ok
CREATE TABLE supplier(
s_suppkey ${type} NOT NULL,
s_name VARCHAR NOT NULL,
s_address VARCHAR NOT NULL,
s_nationkey ${type} NOT NULL,
s_phone VARCHAR NOT NULL,
s_acctbal ${type} NOT NULL,
s_comment VARCHAR NOT NULL
);
statement ok
CREATE TABLE nation(
n_nationkey ${type} NOT NULL,
n_name VARCHAR NOT NULL,
n_regionkey ${type} NOT NULL,
n_comment VARCHAR NOT NULL
);
statement ok
CREATE TABLE region(
r_regionkey ${type} NOT NULL,
r_name VARCHAR NOT NULL,
r_comment VARCHAR NOT NULL
);
# Populate tables
statement ok
insert into lineitem select * from lineitem_original;
statement ok
insert into orders select * from orders_original;
statement ok
insert into partsupp select * from partsupp_original;
statement ok
insert into part select * from part_original;
statement ok
insert into customer select * from customer_original;
statement ok
insert into supplier select * from supplier_original;
statement ok
insert into nation select * from nation_original;
statement ok
insert into region select * from region_original;
# Checkpoint to compress the data
statement ok
checkpoint
# Run the tpch queries
loop i 1 9
query I
PRAGMA tpch(${i})
----
<FILE>:extension/tpch/dbgen/answers/sf1/q0${i}.csv
endloop
loop i 10 15
query I
PRAGMA tpch(${i})
----
<FILE>:extension/tpch/dbgen/answers/sf1/q${i}.csv
endloop
#TPCH 15 - 'sum' replaced with 'kahan_sum'
query I
SELECT
s_suppkey,
s_name,
s_address,
s_phone,
total_revenue
FROM
supplier,
(
SELECT
l_suppkey AS supplier_no,
kahan_sum(l_extendedprice * (1 - l_discount)) AS total_revenue
FROM
lineitem
WHERE
l_shipdate >= CAST('1996-01-01' AS date)
AND l_shipdate < CAST('1996-04-01' AS date)
GROUP BY
supplier_no) revenue0
WHERE
s_suppkey = supplier_no
AND total_revenue = (
SELECT
max(total_revenue)
FROM (
SELECT
l_suppkey AS supplier_no,
kahan_sum(l_extendedprice * (1 - l_discount)) AS total_revenue
FROM
lineitem
WHERE
l_shipdate >= CAST('1996-01-01' AS date)
AND l_shipdate < CAST('1996-04-01' AS date)
GROUP BY
supplier_no) revenue1)
ORDER BY
s_suppkey;
----
<FILE>:extension/tpch/dbgen/answers/sf1/q15.csv
loop i 16 23
query I
PRAGMA tpch(${i})
----
<FILE>:extension/tpch/dbgen/answers/sf1/q${i}.csv
endloop
# Drop tables
statement ok
DROP TABLE lineitem;
statement ok
DROP TABLE orders;
statement ok
DROP TABLE partsupp;
statement ok
DROP TABLE part;
statement ok
DROP TABLE customer;
statement ok
DROP TABLE supplier;
statement ok
DROP TABLE nation;
statement ok
DROP TABLE region;
endloop

44
external/duckdb/test/sql/storage/compression/alp/alp_zeros.test vendored Normal file
View File

@@ -0,0 +1,44 @@
# name: test/sql/storage/compression/alp/alp_zeros.test
# description: Test storage of alp, but simple
# group: [alp]
# load the DB from disk
load __TEST_DIR__/test_alp.db
statement ok
PRAGMA force_compression='uncompressed'
# Create a table with random doubles of limited precision compressed as Uncompressed
# This data should achieve x6 compression ratio
statement ok
create table random_double as select 0::DOUBLE as data from range(1024) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_double') WHERE segment_type == 'double' AND compression != 'Uncompressed';
----
# Now create a duplicate of this table, compressed with ALP instead
statement ok
PRAGMA force_compression='alp'
statement ok
create table random_alp_double as select * from random_double;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_alp_double') WHERE segment_type == 'double' AND compression != 'ALP';
----
# Assert that the data was not corrupted by compressing to ALP
query I sort r1
select * from random_double;
----
query I sort r1
select * from random_alp_double;
----

335
external/duckdb/test/sql/storage/compression/alprd/alprd_inf_null_nan.test vendored Normal file
View File

@@ -0,0 +1,335 @@
# name: test/sql/storage/compression/alprd/alprd_inf_null_nan.test
# group: [alprd]
# load the DB from disk
load __TEST_DIR__/test_alprd_inf.db
foreach compression uncompressed alprd
# Set the compression algorithm
statement ok
pragma force_compression='${compression}'
# Create tables
statement ok
create table tbl1_${compression}(
a INTEGER DEFAULT 5,
b VARCHAR DEFAULT 'test',
c BOOL DEFAULT false,
d DOUBLE,
e TEXT default 'null',
f FLOAT
);
statement ok
create table tbl2_${compression}(
a INTEGER DEFAULT 5,
b VARCHAR DEFAULT 'test',
c BOOL DEFAULT false,
d DOUBLE,
e TEXT default 'null',
f FLOAT
);
statement ok
create table tbl3_${compression}(
a INTEGER DEFAULT 5,
b VARCHAR DEFAULT 'test',
c BOOL DEFAULT false,
d DOUBLE,
e TEXT default 'null',
f FLOAT
);
# Populate tables
# Mixed NULLs
statement ok
insert into tbl1_${compression}(d,f) VALUES
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity');
# Only NULLS
statement ok
insert into tbl2_${compression}(d,f) VALUES
(NULL, NULL),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', 'Infinity'),
('Infinity', '-Infinity'),
('Infinity', '-Infinity'),
('Infinity', '-Infinity'),
('Infinity', '-Infinity'),
('Infinity', '-Infinity'),
('-Infinity', '-Infinity'),
('-Infinity', '-Infinity'),
('-Infinity', '-Infinity'),
('-Infinity', '-Infinity'),
('-Infinity', '-Infinity'),
('-Infinity', '-Infinity'),
(0, 0),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL);
# Starting with NULLS
statement ok
insert into tbl3_${compression}(d,f) VALUES
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', '-Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', '-Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', '-Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', '-Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', '-Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('-Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('Infinity', 'Infinity'),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(NULL, NULL),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
('NaN', 'NaN'),
('NaN', 'NaN'),
('NaN', 'NaN');
# Force a checkpoint
statement ok
checkpoint
endloop
# Assert that the scanned results are the same
#tbl1
query II nosort r1
select d, f from tbl1_uncompressed;
----
query II nosort r1
select d, f from tbl1_alprd;
----
#tbl2
query II nosort r2
select d, f from tbl2_uncompressed;
----
query II nosort r2
select d, f from tbl2_alprd;
----
# tbl3
query II nosort r3
select d, f from tbl3_uncompressed;
----
query II nosort r3
select d, f from tbl3_alprd;
----

44
external/duckdb/test/sql/storage/compression/alprd/alprd_many_segments.test_slow vendored Normal file
View File

@@ -0,0 +1,44 @@
# name: test/sql/storage/compression/alprd/alprd_many_segments.test_slow
# description: Test storage of alprd, but simple
# group: [alprd]
# load the DB from disk
load __TEST_DIR__/test_alprd.db
statement ok
PRAGMA force_compression='uncompressed'
# Create a table with random doubles of limited precision compressed as Uncompressed
# This data should achieve x6 compression ratio
statement ok
create table random_double as select random()::DOUBLE as data from range(500000) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_double') WHERE segment_type == 'double' AND compression != 'Uncompressed';
----
# Now create a duplicate of this table, compressed with ALP instead
statement ok
PRAGMA force_compression='alprd'
statement ok
create table random_alp_double as select * from random_double;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_alp_double') WHERE segment_type == 'double' AND compression != 'ALPRD';
----
# Assert that the data was not corrupted by compressing to ALP
query I sort r1
select * from random_double;
----
query I sort r1
select * from random_alp_double;
----

44
external/duckdb/test/sql/storage/compression/alprd/alprd_many_segments_float.test_slow vendored Normal file
View File

@@ -0,0 +1,44 @@
# name: test/sql/storage/compression/alprd/alprd_many_segments_float.test_slow
# description: Test storage of alprd, but simple
# group: [alprd]
# load the DB from disk
load __TEST_DIR__/test_alprd.db
statement ok
PRAGMA force_compression='uncompressed'
# Create a table with random floats of limited precision compressed as Uncompressed
# This data should achieve x6 compression ratio
statement ok
create table random_float as select random()::FLOAT as data from range(500000) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_float') WHERE segment_type == 'float' AND compression != 'Uncompressed';
----
# Now create a duplicate of this table, compressed with ALPRD instead
statement ok
PRAGMA force_compression='alprd'
statement ok
create table random_alp_float as select * from random_float;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_alp_float') WHERE segment_type == 'float' AND compression != 'ALPRD';
----
# Assert that the data was not corrupted by compressing to ALP
query I sort r1
select * from random_float;
----
query I sort r1
select * from random_alp_float;
----

44
external/duckdb/test/sql/storage/compression/alprd/alprd_middle_flush.test_slow vendored Normal file
View File

@@ -0,0 +1,44 @@
# name: test/sql/storage/compression/alprd/alprd_middle_flush.test_slow
# description: Test storage of alprd, but simple
# group: [alprd]
# load the DB from disk
load __TEST_DIR__/test_alprd.db
statement ok
PRAGMA force_compression='uncompressed'
# Create a table with random doubles of limited precision compressed as Uncompressed
# This data should achieve x6 compression ratio
statement ok
create table random_double as select random()::DOUBLE as data from range(110000) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_double') WHERE segment_type == 'double' AND compression != 'Uncompressed';
----
# Now create a duplicate of this table, compressed with ALPRD instead
statement ok
PRAGMA force_compression='alprd'
statement ok
create table random_alp_double as select * from random_double;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_alp_double') WHERE segment_type == 'double' AND compression != 'ALPRD';
----
# Assert that the data was not corrupted by compressing to ALP
query I sort r1
select * from random_double;
----
query I sort r1
select * from random_alp_double;
----

37
external/duckdb/test/sql/storage/compression/alprd/alprd_min_max.test vendored Normal file
View File

@@ -0,0 +1,37 @@
# name: test/sql/storage/compression/alprd/alprd_min_max.test
# group: [alprd]
load __TEST_DIR__/alprd_min_max.db
statement ok
PRAGMA enable_verification
statement ok
PRAGMA force_compression='alprd';
foreach type DOUBLE FLOAT
statement ok
CREATE TABLE all_types AS SELECT ${type} FROM test_all_types();
loop i 0 15
statement ok
INSERT INTO all_types SELECT ${type} FROM all_types;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('all_types') WHERE segment_type == '${type}' AND compression != 'ALPRD';
----
# i
endloop
statement ok
DROP TABLE all_types;
#type
endloop

44
external/duckdb/test/sql/storage/compression/alprd/alprd_negative_numbers.test vendored Normal file
View File

@@ -0,0 +1,44 @@
# name: test/sql/storage/compression/alprd/alprd_negative_numbers.test
# description: Test storage of alprd, but simple
# group: [alprd]
# load the DB from disk
load __TEST_DIR__/test_alprd.db
statement ok
PRAGMA force_compression='uncompressed'
# Create a table with random doubles of limited precision compressed as Uncompressed
# This data should achieve x6 compression ratio
statement ok
create table random_double as select round(cos(1 / (random() + 0.001)), 15)::DOUBLE * -1 as data from range(1024) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_double') WHERE segment_type == 'double' AND compression != 'Uncompressed';
----
# Now create a duplicate of this table, compressed with ALP instead
statement ok
PRAGMA force_compression='alprd'
statement ok
create table random_alp_double as select * from random_double;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_alp_double') WHERE segment_type == 'double' AND compression != 'ALPRD';
----
# Assert that the data was not corrupted by compressing to ALP
query I sort r1
select * from random_double;
----
query I sort r1
select * from random_alp_double;
----

343
external/duckdb/test/sql/storage/compression/alprd/alprd_nulls.test vendored Normal file
View File

@@ -0,0 +1,343 @@
# name: test/sql/storage/compression/alprd/alprd_nulls.test
# group: [alprd]
# load the DB from disk
load __TEST_DIR__/test_alprd_nulls.db
foreach compression uncompressed alprd
# Set the compression algorithm
statement ok
pragma force_compression='${compression}'
# Create tables
statement ok
create table tbl1_${compression}(
a INTEGER DEFAULT 5,
b VARCHAR DEFAULT 'test',
c BOOL DEFAULT false,
d DOUBLE,
e TEXT default 'null',
f FLOAT
);
statement ok
create table tbl2_${compression}(
a INTEGER DEFAULT 5,
b VARCHAR DEFAULT 'test',
c BOOL DEFAULT false,
d DOUBLE,
e TEXT default 'null',
f FLOAT
);
statement ok
create table tbl3_${compression}(
a INTEGER DEFAULT 5,
b VARCHAR DEFAULT 'test',
c BOOL DEFAULT false,
d DOUBLE,
e TEXT default 'null',
f FLOAT
);
# Populate tables
# Mixed NULLs
statement ok
insert into tbl1_${compression}(d,f) VALUES
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, NULL);
# Only NULLS
statement ok
insert into tbl2_${compression}(d,f) VALUES
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL);
# Starting with NULLS
statement ok
insert into tbl3_${compression}(d,f) VALUES
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(NULL, NULL),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142),
(7034.34968234, 93472948.98034),
(1.213123, 1.232142);
# Force a checkpoint
statement ok
checkpoint
endloop
# Assert that the scanned results are the same
#tbl1
query II nosort r1
select d, f from tbl1_uncompressed;
----
query II nosort r1
select d, f from tbl1_alprd;
----
#tbl2
query II nosort r2
select d, f from tbl2_uncompressed;
----
query II nosort r2
select d, f from tbl2_alprd;
----
# tbl3
query II nosort r3
select d, f from tbl3_uncompressed;
----
query II nosort r3
select d, f from tbl3_alprd;
----

129
external/duckdb/test/sql/storage/compression/alprd/alprd_nulls_simple.test vendored Normal file
View File

@@ -0,0 +1,129 @@
# name: test/sql/storage/compression/alprd/alprd_nulls_simple.test
# group: [alprd]
# load the DB from disk
load __TEST_DIR__/test_alprd_nulls.db
foreach compression uncompressed alprd
# Set the compression algorithm
statement ok
pragma force_compression='${compression}'
# Create tables
statement ok
create table tbl1_${compression}(
a INTEGER DEFAULT 5,
b VARCHAR DEFAULT 'test',
c BOOL DEFAULT false,
d DOUBLE,
e TEXT default 'null',
f FLOAT
);
# Populate tables
# Mixed NULLs
statement ok
insert into tbl1_${compression}(d,f) VALUES
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, 1.2314234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(324213.23123, NULL),
(NULL, NULL),
(21312.23412, 12.123234),
(NULL, NULL);
# Force a checkpoint
statement ok
checkpoint
endloop
# Assert that the scanned results are the same
#tbl1
query II nosort r1
select d, f from tbl1_uncompressed;
----
query II nosort r1
select d, f from tbl1_alprd;
----

51
external/duckdb/test/sql/storage/compression/alprd/alprd_read.test_slow vendored Normal file
View File

@@ -0,0 +1,51 @@
# name: test/sql/storage/compression/alprd/alprd_read.test_slow
# group: [alprd]
require parquet
require httpfs
load __TEST_DIR__/test_alprd.db
statement ok
DROP TABLE IF EXISTS temperatures;
statement ok
pragma threads=1;
statement ok
PRAGMA force_compression='alprd';
statement ok
CREATE TABLE temperatures (
temperature DOUBLE
);
statement ok
INSERT INTO temperatures SELECT temp from 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/city_temperature.parquet' t(temp), range(1);
statement ok
CHECKPOINT;
statement ok
create table reference_temperatures (
temperature DOUBLE
);
statement ok
PRAGMA force_compression='uncompressed';
statement ok
INSERT INTO reference_temperatures SELECT temp from 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/city_temperature.parquet' t(temp), range(1);
statement ok
checkpoint;
query I nosort r1
select temperature from reference_temperatures;
----
query I nosort r1
select temperature from temperatures
----

43
external/duckdb/test/sql/storage/compression/alprd/alprd_simple.test vendored Normal file
View File

@@ -0,0 +1,43 @@
# name: test/sql/storage/compression/alprd/alprd_simple.test
# description: Test storage of alprd, but simple
# group: [alprd]
# load the DB from disk
load __TEST_DIR__/test_alprd.db
statement ok
PRAGMA force_compression='uncompressed'
# Create a table with random doubles of limited precision compressed as Uncompressed
statement ok
create table random_double as select random()::DOUBLE as data from range(1024) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_double') WHERE segment_type == 'double' AND compression != 'Uncompressed';
----
# Now create a duplicate of this table, compressed with ALPRD instead
statement ok
PRAGMA force_compression='alprd'
statement ok
create table random_alprd_double as select * from random_double;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_alprd_double') WHERE segment_type == 'double' AND compression != 'ALPRD';
----
# Assert that the data was not corrupted by compressing to ALPRD
query I sort r1
select * from random_double;
----
query I sort r1
select * from random_alprd_double;
----

43
external/duckdb/test/sql/storage/compression/alprd/alprd_simple_float.test vendored Normal file
View File

@@ -0,0 +1,43 @@
# name: test/sql/storage/compression/alprd/alprd_simple_float.test
# description: Test storage of alprd, but simple
# group: [alprd]
# load the DB from disk
load __TEST_DIR__/test_alp.db
statement ok
PRAGMA force_compression='uncompressed'
# Create a table with random floats of limited precision compressed as Uncompressed
statement ok
create table random_float as select random()::FLOAT as data from range(1024) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_float') WHERE segment_type == 'float' AND compression != 'Uncompressed';
----
# Now create a duplicate of this table, compressed with ALP instead
statement ok
PRAGMA force_compression='alprd'
statement ok
create table random_alp_float as select * from random_float;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_alp_float') WHERE segment_type == 'float' AND compression != 'ALPRD';
----
# Assert that the data was not corrupted by compressing to ALP
query I sort r1
select * from random_float;
----
query I sort r1
select * from random_alp_float;
----

55
external/duckdb/test/sql/storage/compression/alprd/alprd_skip.test_slow vendored Normal file
View File

@@ -0,0 +1,55 @@
# name: test/sql/storage/compression/alprd/alprd_skip.test_slow
# group: [alprd]
# load the DB from disk
load __TEST_DIR__/test_alprd.db
statement ok
pragma enable_verification;
statement ok
pragma disable_optimizer;
statement ok
pragma force_compression='uncompressed'
# Create the data for the columns
statement ok
create table temp_table as select random()::DOUBLE * 100 as col, j from range(10240) tbl(j);
statement ok
checkpoint
foreach compression ALPRD Uncompressed
# Ensure the correct compression is used
statement ok
pragma force_compression='${compression}'
# Setup
statement ok
create table tbl_${compression} as select * from temp_table;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('tbl_${compression}') WHERE segment_type == 'double' AND compression != '${compression}';
----
# compression
endloop
loop i 1 1024
query II
select x as x_${i}, y as y_${i} from (
select
(select col from tbl_alprd where (j > (${i} * 1024)) except select col from tbl_uncompressed where (j > (${i} * 1024))) as x,
(select col from tbl_uncompressed where (j > (${i} * 1024)) except select col from tbl_alprd where (j > (${i} * 1024))) as y
);
----
NULL NULL
# i
endloop

94
external/duckdb/test/sql/storage/compression/alprd/alprd_stress_test.test_slow vendored Normal file
View File

@@ -0,0 +1,94 @@
# name: test/sql/storage/compression/alprd/alprd_stress_test.test_slow
# group: [alprd]
load __TEST_DIR__/alprd_min_max.db
foreach type DOUBLE FLOAT
statement ok
pragma force_compression='uncompressed'
# Create the initial data to start with (without this Constant compression will be used at first)
statement ok
create table temp_table as select random()::${type} * 100 from range(5);
statement ok
checkpoint
foreach compression ALPRD Uncompressed
# Ensure the correct compression is used
statement ok
pragma force_compression='${compression}'
# Setup
statement ok
create table ${compression}_tbl(
data ${type}
);
statement ok
insert into ${compression}_tbl select * from temp_table;
statement ok
checkpoint
# compression
endloop
statement ok
drop table temp_table;
# ---------------- MAIN LOOP ----------------
loop i 0 10240
# Create a temporary table containing the value we want to add to both tables
statement ok
create table temp_table as select random()::${type} * 100 + ${i};
foreach compression ALPRD Uncompressed
# Ensure the correct compression is used
statement ok
pragma force_compression='${compression}'
# Insert a single value into the table
statement ok
insert into ${compression}_tbl select * from temp_table;
# Checkpoint the table with the newly added data
statement ok
checkpoint
# compression
endloop
statement ok
drop table temp_table;
# ---------------- VERIFY CORRECTNESS ----------------
query II
select x as x_${i}, y as y_${i} from (
select
(select data from alprd_tbl except select data from uncompressed_tbl) as x,
(select data from uncompressed_tbl except select data from alprd_tbl) as y
);
----
NULL NULL
# i
endloop
# Teardown
foreach compression alprd uncompressed
statement ok
drop table ${compression}_tbl;
# compression
endloop
# type
endloop

753
external/duckdb/test/sql/storage/compression/alprd/alprd_tpcds.test_slow vendored Normal file
View File

@@ -0,0 +1,753 @@
# name: test/sql/storage/compression/alprd/alprd_tpcds.test_slow
# group: [alprd]
require tpcds
# load the DB from disk
load __TEST_DIR__/test_alprd.db
#statement ok
#pragma threads=1
statement ok
PRAGMA force_compression='alprd';
statement ok
call dsdgen(sf=1, suffix='_original');
statement ok
PRAGMA default_null_order='NULLS LAST'
# Test both DOUBLE and FLOAT
foreach type DOUBLE FLOAT
# Create tables
statement ok
CREATE TABLE web_site(
web_site_sk ${type},
web_site_id VARCHAR,
web_rec_start_date DATE,
web_rec_end_date DATE,
web_name VARCHAR,
web_open_date_sk ${type},
web_close_date_sk ${type},
web_class VARCHAR,
web_manager VARCHAR,
web_mkt_id ${type},
web_mkt_class VARCHAR,
web_mkt_desc VARCHAR,
web_market_manager VARCHAR,
web_company_id ${type},
web_company_name VARCHAR,
web_street_number VARCHAR,
web_street_name VARCHAR,
web_street_type VARCHAR,
web_suite_number VARCHAR,
web_city VARCHAR,
web_county VARCHAR,
web_state VARCHAR,
web_zip VARCHAR,
web_country VARCHAR,
web_gmt_offset ${type},
web_tax_percentage ${type}
);
statement ok
CREATE TABLE web_sales(
ws_sold_date_sk ${type},
ws_sold_time_sk ${type},
ws_ship_date_sk ${type},
ws_item_sk ${type},
ws_bill_customer_sk ${type},
ws_bill_cdemo_sk ${type},
ws_bill_hdemo_sk ${type},
ws_bill_addr_sk ${type},
ws_ship_customer_sk ${type},
ws_ship_cdemo_sk ${type},
ws_ship_hdemo_sk ${type},
ws_ship_addr_sk ${type},
ws_web_page_sk ${type},
ws_web_site_sk ${type},
ws_ship_mode_sk ${type},
ws_warehouse_sk ${type},
ws_promo_sk ${type},
ws_order_number ${type},
ws_quantity ${type},
ws_wholesale_cost ${type},
ws_list_price ${type},
ws_sales_price ${type},
ws_ext_discount_amt ${type},
ws_ext_sales_price ${type},
ws_ext_wholesale_cost ${type},
ws_ext_list_price ${type},
ws_ext_tax ${type},
ws_coupon_amt ${type},
ws_ext_ship_cost ${type},
ws_net_paid ${type},
ws_net_paid_inc_tax ${type},
ws_net_paid_inc_ship ${type},
ws_net_paid_inc_ship_tax ${type},
ws_net_profit ${type}
);
statement ok
CREATE TABLE web_returns(
wr_returned_date_sk ${type},
wr_returned_time_sk ${type},
wr_item_sk ${type},
wr_refunded_customer_sk ${type},
wr_refunded_cdemo_sk ${type},
wr_refunded_hdemo_sk ${type},
wr_refunded_addr_sk ${type},
wr_returning_customer_sk ${type},
wr_returning_cdemo_sk ${type},
wr_returning_hdemo_sk ${type},
wr_returning_addr_sk ${type},
wr_web_page_sk ${type},
wr_reason_sk ${type},
wr_order_number ${type},
wr_return_quantity ${type},
wr_return_amt ${type},
wr_return_tax ${type},
wr_return_amt_inc_tax ${type},
wr_fee ${type},
wr_return_ship_cost ${type},
wr_refunded_cash ${type},
wr_reversed_charge ${type},
wr_account_credit ${type},
wr_net_loss ${type}
);
statement ok
CREATE TABLE web_page(
wp_web_page_sk ${type},
wp_web_page_id VARCHAR,
wp_rec_start_date DATE,
wp_rec_end_date DATE,
wp_creation_date_sk ${type},
wp_access_date_sk ${type},
wp_autogen_flag VARCHAR,
wp_customer_sk ${type},
wp_url VARCHAR,
wp_type VARCHAR,
wp_char_count ${type},
wp_link_count ${type},
wp_image_count ${type},
wp_max_ad_count ${type}
);
statement ok
CREATE TABLE warehouse(
w_warehouse_sk ${type},
w_warehouse_id VARCHAR,
w_warehouse_name VARCHAR,
w_warehouse_sq_ft ${type},
w_street_number VARCHAR,
w_street_name VARCHAR,
w_street_type VARCHAR,
w_suite_number VARCHAR,
w_city VARCHAR,
w_county VARCHAR,
w_state VARCHAR,
w_zip VARCHAR,
w_country VARCHAR,
w_gmt_offset ${type}
);
statement ok
CREATE TABLE time_dim(
t_time_sk ${type},
t_time_id VARCHAR,
t_time ${type},
t_hour ${type},
t_minute ${type},
t_second ${type},
t_am_pm VARCHAR,
t_shift VARCHAR,
t_sub_shift VARCHAR,
t_meal_time VARCHAR
);
statement ok
CREATE TABLE store_sales(
ss_sold_date_sk ${type},
ss_sold_time_sk ${type},
ss_item_sk ${type},
ss_customer_sk ${type},
ss_cdemo_sk ${type},
ss_hdemo_sk ${type},
ss_addr_sk ${type},
ss_store_sk ${type},
ss_promo_sk ${type},
ss_ticket_number ${type},
ss_quantity ${type},
ss_wholesale_cost ${type},
ss_list_price ${type},
ss_sales_price ${type},
ss_ext_discount_amt ${type},
ss_ext_sales_price ${type},
ss_ext_wholesale_cost ${type},
ss_ext_list_price ${type},
ss_ext_tax ${type},
ss_coupon_amt ${type},
ss_net_paid ${type},
ss_net_paid_inc_tax ${type},
ss_net_profit ${type}
);
statement ok
CREATE TABLE store_returns(
sr_returned_date_sk ${type},
sr_return_time_sk ${type},
sr_item_sk ${type},
sr_customer_sk ${type},
sr_cdemo_sk ${type},
sr_hdemo_sk ${type},
sr_addr_sk ${type},
sr_store_sk ${type},
sr_reason_sk ${type},
sr_ticket_number ${type},
sr_return_quantity ${type},
sr_return_amt ${type},
sr_return_tax ${type},
sr_return_amt_inc_tax ${type},
sr_fee ${type},
sr_return_ship_cost ${type},
sr_refunded_cash ${type},
sr_reversed_charge ${type},
sr_store_credit ${type},
sr_net_loss ${type}
);
statement ok
CREATE TABLE store(
s_store_sk ${type},
s_store_id VARCHAR,
s_rec_start_date DATE,
s_rec_end_date DATE,
s_closed_date_sk ${type},
s_store_name VARCHAR,
s_number_employees ${type},
s_floor_space ${type},
s_hours VARCHAR,
s_manager VARCHAR,
s_market_id ${type},
s_geography_class VARCHAR,
s_market_desc VARCHAR,
s_market_manager VARCHAR,
s_division_id ${type},
s_division_name VARCHAR,
s_company_id ${type},
s_company_name VARCHAR,
s_street_number VARCHAR,
s_street_name VARCHAR,
s_street_type VARCHAR,
s_suite_number VARCHAR,
s_city VARCHAR,
s_county VARCHAR,
s_state VARCHAR,
s_zip VARCHAR,
s_country VARCHAR,
s_gmt_offset ${type},
s_tax_percentage ${type}
);
statement ok
CREATE TABLE ship_mode(
sm_ship_mode_sk ${type},
sm_ship_mode_id VARCHAR,
sm_type VARCHAR,
sm_code VARCHAR,
sm_carrier VARCHAR,
sm_contract VARCHAR
);
statement ok
CREATE TABLE reason(
r_reason_sk ${type},
r_reason_id VARCHAR,
r_reason_desc VARCHAR
);
statement ok
CREATE TABLE promotion(
p_promo_sk ${type},
p_promo_id VARCHAR,
p_start_date_sk ${type},
p_end_date_sk ${type},
p_item_sk ${type},
p_cost ${type},
p_response_target ${type},
p_promo_name VARCHAR,
p_channel_dmail VARCHAR,
p_channel_email VARCHAR,
p_channel_catalog VARCHAR,
p_channel_tv VARCHAR,
p_channel_radio VARCHAR,
p_channel_press VARCHAR,
p_channel_event VARCHAR,
p_channel_demo VARCHAR,
p_channel_details VARCHAR,
p_purpose VARCHAR,
p_discount_active VARCHAR
);
statement ok
CREATE TABLE item(
i_item_sk ${type},
i_item_id VARCHAR,
i_rec_start_date DATE,
i_rec_end_date DATE,
i_item_desc VARCHAR,
i_current_price ${type},
i_wholesale_cost ${type},
i_brand_id ${type},
i_brand VARCHAR,
i_class_id ${type},
i_class VARCHAR,
i_category_id ${type},
i_category VARCHAR,
i_manufact_id ${type},
i_manufact VARCHAR,
i_size VARCHAR,
i_formulation VARCHAR,
i_color VARCHAR,
i_units VARCHAR,
i_container VARCHAR,
i_manager_id ${type},
i_product_name VARCHAR
);
statement ok
CREATE TABLE inventory(
inv_date_sk ${type},
inv_item_sk ${type},
inv_warehouse_sk ${type},
inv_quantity_on_hand ${type}
);
statement ok
CREATE TABLE income_band(
ib_income_band_sk ${type},
ib_lower_bound ${type},
ib_upper_bound ${type}
);
statement ok
CREATE TABLE household_demographics(
hd_demo_sk ${type},
hd_income_band_sk ${type},
hd_buy_potential VARCHAR,
hd_dep_count ${type},
hd_vehicle_count ${type}
);
statement ok
CREATE TABLE date_dim(
d_date_sk ${type},
d_date_id VARCHAR,
d_date DATE,
d_month_seq ${type},
d_week_seq ${type},
d_quarter_seq ${type},
d_year ${type},
d_dow ${type},
d_moy ${type},
d_dom ${type},
d_qoy ${type},
d_fy_year ${type},
d_fy_quarter_seq ${type},
d_fy_week_seq ${type},
d_day_name VARCHAR,
d_quarter_name VARCHAR,
d_holiday VARCHAR,
d_weekend VARCHAR,
d_following_holiday VARCHAR,
d_first_dom ${type},
d_last_dom ${type},
d_same_day_ly ${type},
d_same_day_lq ${type},
d_current_day VARCHAR,
d_current_week VARCHAR,
d_current_month VARCHAR,
d_current_quarter VARCHAR,
d_current_year VARCHAR
);
statement ok
CREATE TABLE customer_demographics(
cd_demo_sk ${type},
cd_gender VARCHAR,
cd_marital_status VARCHAR,
cd_education_status VARCHAR,
cd_purchase_estimate ${type},
cd_credit_rating VARCHAR,
cd_dep_count ${type},
cd_dep_employed_count ${type},
cd_dep_college_count ${type}
);
statement ok
CREATE TABLE customer_address(
ca_address_sk ${type},
ca_address_id VARCHAR,
ca_street_number VARCHAR,
ca_street_name VARCHAR,
ca_street_type VARCHAR,
ca_suite_number VARCHAR,
ca_city VARCHAR,
ca_county VARCHAR,
ca_state VARCHAR,
ca_zip VARCHAR,
ca_country VARCHAR,
ca_gmt_offset ${type},
ca_location_type VARCHAR
);
statement ok
CREATE TABLE customer(
c_customer_sk ${type},
c_customer_id VARCHAR,
c_current_cdemo_sk ${type},
c_current_hdemo_sk ${type},
c_current_addr_sk ${type},
c_first_shipto_date_sk ${type},
c_first_sales_date_sk ${type},
c_salutation VARCHAR,
c_first_name VARCHAR,
c_last_name VARCHAR,
c_preferred_cust_flag VARCHAR,
c_birth_day ${type},
c_birth_month ${type},
c_birth_year ${type},
c_birth_country VARCHAR,
c_login VARCHAR,
c_email_address VARCHAR,
c_last_review_date_sk ${type}
);
statement ok
CREATE TABLE catalog_sales(
cs_sold_date_sk ${type},
cs_sold_time_sk ${type},
cs_ship_date_sk ${type},
cs_bill_customer_sk ${type},
cs_bill_cdemo_sk ${type},
cs_bill_hdemo_sk ${type},
cs_bill_addr_sk ${type},
cs_ship_customer_sk ${type},
cs_ship_cdemo_sk ${type},
cs_ship_hdemo_sk ${type},
cs_ship_addr_sk ${type},
cs_call_center_sk ${type},
cs_catalog_page_sk ${type},
cs_ship_mode_sk ${type},
cs_warehouse_sk ${type},
cs_item_sk ${type},
cs_promo_sk ${type},
cs_order_number ${type},
cs_quantity ${type},
cs_wholesale_cost ${type},
cs_list_price ${type},
cs_sales_price ${type},
cs_ext_discount_amt ${type},
cs_ext_sales_price ${type},
cs_ext_wholesale_cost ${type},
cs_ext_list_price ${type},
cs_ext_tax ${type},
cs_coupon_amt ${type},
cs_ext_ship_cost ${type},
cs_net_paid ${type},
cs_net_paid_inc_tax ${type},
cs_net_paid_inc_ship ${type},
cs_net_paid_inc_ship_tax ${type},
cs_net_profit ${type}
);
statement ok
CREATE TABLE catalog_returns(
cr_returned_date_sk ${type},
cr_returned_time_sk ${type},
cr_item_sk ${type},
cr_refunded_customer_sk ${type},
cr_refunded_cdemo_sk ${type},
cr_refunded_hdemo_sk ${type},
cr_refunded_addr_sk ${type},
cr_returning_customer_sk ${type},
cr_returning_cdemo_sk ${type},
cr_returning_hdemo_sk ${type},
cr_returning_addr_sk ${type},
cr_call_center_sk ${type},
cr_catalog_page_sk ${type},
cr_ship_mode_sk ${type},
cr_warehouse_sk ${type},
cr_reason_sk ${type},
cr_order_number ${type},
cr_return_quantity ${type},
cr_return_amount ${type},
cr_return_tax ${type},
cr_return_amt_inc_tax ${type},
cr_fee ${type},
cr_return_ship_cost ${type},
cr_refunded_cash ${type},
cr_reversed_charge ${type},
cr_store_credit ${type},
cr_net_loss ${type}
);
statement ok
CREATE TABLE catalog_page(
cp_catalog_page_sk ${type},
cp_catalog_page_id VARCHAR,
cp_start_date_sk ${type},
cp_end_date_sk ${type},
cp_department VARCHAR,
cp_catalog_number ${type},
cp_catalog_page_number ${type},
cp_description VARCHAR,
cp_type VARCHAR
);
statement ok
CREATE TABLE call_center(
cc_call_center_sk ${type},
cc_call_center_id VARCHAR,
cc_rec_start_date DATE,
cc_rec_end_date DATE,
cc_closed_date_sk ${type},
cc_open_date_sk ${type},
cc_name VARCHAR,
cc_class VARCHAR,
cc_employees ${type},
cc_sq_ft ${type},
cc_hours VARCHAR,
cc_manager VARCHAR,
cc_mkt_id ${type},
cc_mkt_class VARCHAR,
cc_mkt_desc VARCHAR,
cc_market_manager VARCHAR,
cc_division ${type},
cc_division_name VARCHAR,
cc_company ${type},
cc_company_name VARCHAR,
cc_street_number VARCHAR,
cc_street_name VARCHAR,
cc_street_type VARCHAR,
cc_suite_number VARCHAR,
cc_city VARCHAR,
cc_county VARCHAR,
cc_state VARCHAR,
cc_zip VARCHAR,
cc_country VARCHAR,
cc_gmt_offset ${type},
cc_tax_percentage ${type}
);
# Populate tables
statement ok
insert into web_site select * from web_site_original;
statement ok
insert into web_sales select * from web_sales_original;
statement ok
insert into web_returns select * from web_returns_original;
statement ok
insert into web_page select * from web_page_original;
statement ok
insert into warehouse select * from warehouse_original;
statement ok
insert into time_dim select * from time_dim_original;
statement ok
insert into store_sales select * from store_sales_original;
statement ok
insert into store_returns select * from store_returns_original;
statement ok
insert into store select * from store_original;
statement ok
insert into ship_mode select * from ship_mode_original;
statement ok
insert into reason select * from reason_original;
statement ok
insert into promotion select * from promotion_original;
statement ok
insert into item select * from item_original;
statement ok
insert into inventory select * from inventory_original;
statement ok
insert into income_band select * from income_band_original;
statement ok
insert into household_demographics select * from household_demographics_original;
statement ok
insert into date_dim select * from date_dim_original;
statement ok
insert into customer_demographics select * from customer_demographics_original;
statement ok
insert into customer_address select * from customer_address_original;
statement ok
insert into customer select * from customer_original;
statement ok
insert into catalog_sales select * from catalog_sales_original;
statement ok
insert into catalog_returns select * from catalog_returns_original;
statement ok
insert into catalog_page select * from catalog_page_original;
statement ok
insert into call_center select * from call_center_original;
# Checkpoint to compress the data
statement ok
checkpoint
# And verify that no other compression is used
foreach tbl web_site web_sales web_returns web_page warehouse time_dim store_sales store_returns store ship_mode reason promotion item inventory income_band household_demographics date_dim customer_demographics customer_address customer catalog_sales catalog_returns catalog_page call_center
# Cant turn off the creation of constant segments, so we have to just accept that some of the segments are Constant
query I
SELECT compression FROM pragma_storage_info('${tbl}') WHERE segment_type == '${type}' AND compression != 'ALPRD' AND compression != 'Constant';
----
endloop
# Run the tpcds queries
loop i 1 9
query I
PRAGMA tpcds(${i})
----
<FILE>:extension/tpcds/dsdgen/answers/sf1/0${i}.csv
endloop
loop i 10 49
#Skip tpcds 49 because it doesn't work without decimals
query I
PRAGMA tpcds(${i})
----
<FILE>:extension/tpcds/dsdgen/answers/sf1/${i}.csv
endloop
# skip tpcds 67 - inconsistent without decimals
loop i 50 66
query I
PRAGMA tpcds(${i})
----
<FILE>:extension/tpcds/dsdgen/answers/sf1/${i}.csv
endloop
loop i 68 99
query I
PRAGMA tpcds(${i})
----
<FILE>:extension/tpcds/dsdgen/answers/sf1/${i}.csv
endloop
# Drop tables
statement ok
DROP TABLE web_site;
statement ok
DROP TABLE web_sales;
statement ok
DROP TABLE web_returns;
statement ok
DROP TABLE web_page;
statement ok
DROP TABLE warehouse;
statement ok
DROP TABLE time_dim;
statement ok
DROP TABLE store_sales;
statement ok
DROP TABLE store_returns;
statement ok
DROP TABLE store;
statement ok
DROP TABLE ship_mode;
statement ok
DROP TABLE reason;
statement ok
DROP TABLE promotion;
statement ok
DROP TABLE item;
statement ok
DROP TABLE inventory;
statement ok
DROP TABLE income_band;
statement ok
DROP TABLE household_demographics;
statement ok
DROP TABLE date_dim;
statement ok
DROP TABLE customer_demographics;
statement ok
DROP TABLE customer_address;
statement ok
DROP TABLE customer;
statement ok
DROP TABLE catalog_sales;
statement ok
DROP TABLE catalog_returns;
statement ok
DROP TABLE catalog_page;
statement ok
DROP TABLE call_center;
endloop

247
external/duckdb/test/sql/storage/compression/alprd/alprd_tpch.test_slow vendored Normal file
View File

@@ -0,0 +1,247 @@
# name: test/sql/storage/compression/alprd/alprd_tpch.test_slow
# group: [alprd]
require tpch
# load the DB from disk
load __TEST_DIR__/test_alprd.db
# This needs to be single-threaded to be consistent (because of floating point issues)
statement ok
pragma threads=1
statement ok
PRAGMA force_compression='alprd';
statement ok
call dbgen(sf=1, suffix='_original');
# Test both DOUBLE and FLOAT
foreach type DOUBLE FLOAT
# Create tables
statement ok
CREATE TABLE lineitem(
l_orderkey ${type} NOT NULL,
l_partkey ${type} NOT NULL,
l_suppkey ${type} NOT NULL,
l_linenumber ${type} NOT NULL,
l_quantity ${type} NOT NULL,
l_extendedprice ${type} NOT NULL,
l_discount ${type} NOT NULL,
l_tax ${type} NOT NULL,
l_returnflag VARCHAR NOT NULL,
l_linestatus VARCHAR NOT NULL,
l_shipdate DATE NOT NULL,
l_commitdate DATE NOT NULL,
l_receiptdate DATE NOT NULL,
l_shipinstruct VARCHAR NOT NULL,
l_shipmode VARCHAR NOT NULL,
l_comment VARCHAR NOT NULL
);
statement ok
CREATE TABLE orders(
o_orderkey ${type} NOT NULL,
o_custkey ${type} NOT NULL,
o_orderstatus VARCHAR NOT NULL,
o_totalprice ${type} NOT NULL,
o_orderdate DATE NOT NULL,
o_orderpriority VARCHAR NOT NULL,
o_clerk VARCHAR NOT NULL,
o_shippriority ${type} NOT NULL,
o_comment VARCHAR NOT NULL
);
statement ok
CREATE TABLE partsupp(
ps_partkey ${type} NOT NULL,
ps_suppkey ${type} NOT NULL,
ps_availqty ${type} NOT NULL,
ps_supplycost ${type} NOT NULL,
ps_comment VARCHAR NOT NULL
);
# 'p_partkey' being INTEGER is imperative to TPCH(17)
statement ok
CREATE TABLE part(
p_partkey INTEGER NOT NULL,
p_name VARCHAR NOT NULL,
p_mfgr VARCHAR NOT NULL,
p_brand VARCHAR NOT NULL,
p_type VARCHAR NOT NULL,
p_size ${type} NOT NULL,
p_container VARCHAR NOT NULL,
p_retailprice ${type} NOT NULL,
p_comment VARCHAR NOT NULL
);
statement ok
CREATE TABLE customer(
c_custkey ${type} NOT NULL,
c_name VARCHAR NOT NULL,
c_address VARCHAR NOT NULL,
c_nationkey ${type} NOT NULL,
c_phone VARCHAR NOT NULL,
c_acctbal ${type} NOT NULL,
c_mktsegment VARCHAR NOT NULL,
c_comment VARCHAR NOT NULL
);
statement ok
CREATE TABLE supplier(
s_suppkey ${type} NOT NULL,
s_name VARCHAR NOT NULL,
s_address VARCHAR NOT NULL,
s_nationkey ${type} NOT NULL,
s_phone VARCHAR NOT NULL,
s_acctbal ${type} NOT NULL,
s_comment VARCHAR NOT NULL
);
statement ok
CREATE TABLE nation(
n_nationkey ${type} NOT NULL,
n_name VARCHAR NOT NULL,
n_regionkey ${type} NOT NULL,
n_comment VARCHAR NOT NULL
);
statement ok
CREATE TABLE region(
r_regionkey ${type} NOT NULL,
r_name VARCHAR NOT NULL,
r_comment VARCHAR NOT NULL
);
# Populate tables
statement ok
insert into lineitem select * from lineitem_original;
statement ok
insert into orders select * from orders_original;
statement ok
insert into partsupp select * from partsupp_original;
statement ok
insert into part select * from part_original;
statement ok
insert into customer select * from customer_original;
statement ok
insert into supplier select * from supplier_original;
statement ok
insert into nation select * from nation_original;
statement ok
insert into region select * from region_original;
# Checkpoint to compress the data
statement ok
checkpoint
# Run the tpch queries
loop i 1 9
query I
PRAGMA tpch(${i})
----
<FILE>:extension/tpch/dbgen/answers/sf1/q0${i}.csv
endloop
loop i 10 15
query I
PRAGMA tpch(${i})
----
<FILE>:extension/tpch/dbgen/answers/sf1/q${i}.csv
endloop
#TPCH 15 - 'sum' replaced with 'kahan_sum'
query I
SELECT
s_suppkey,
s_name,
s_address,
s_phone,
total_revenue
FROM
supplier,
(
SELECT
l_suppkey AS supplier_no,
kahan_sum(l_extendedprice * (1 - l_discount)) AS total_revenue
FROM
lineitem
WHERE
l_shipdate >= CAST('1996-01-01' AS date)
AND l_shipdate < CAST('1996-04-01' AS date)
GROUP BY
supplier_no) revenue0
WHERE
s_suppkey = supplier_no
AND total_revenue = (
SELECT
max(total_revenue)
FROM (
SELECT
l_suppkey AS supplier_no,
kahan_sum(l_extendedprice * (1 - l_discount)) AS total_revenue
FROM
lineitem
WHERE
l_shipdate >= CAST('1996-01-01' AS date)
AND l_shipdate < CAST('1996-04-01' AS date)
GROUP BY
supplier_no) revenue1)
ORDER BY
s_suppkey;
----
<FILE>:extension/tpch/dbgen/answers/sf1/q15.csv
loop i 16 23
query I
PRAGMA tpch(${i})
----
<FILE>:extension/tpch/dbgen/answers/sf1/q${i}.csv
endloop
# Drop tables
statement ok
DROP TABLE lineitem;
statement ok
DROP TABLE orders;
statement ok
DROP TABLE partsupp;
statement ok
DROP TABLE part;
statement ok
DROP TABLE customer;
statement ok
DROP TABLE supplier;
statement ok
DROP TABLE nation;
statement ok
DROP TABLE region;
endloop

43
external/duckdb/test/sql/storage/compression/alprd/alprd_zeros.test vendored Normal file
View File

@@ -0,0 +1,43 @@
# name: test/sql/storage/compression/alprd/alprd_zeros.test
# description: Test storage of alprd, but simple
# group: [alprd]
# load the DB from disk
load __TEST_DIR__/test_alprd.db
statement ok
PRAGMA force_compression='uncompressed'
# Create a table with random doubles of limited precision compressed as Uncompressed
statement ok
create table random_double as select 0::DOUBLE as data from range(1024) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_double') WHERE segment_type == 'double' AND compression != 'Uncompressed';
----
# Now create a duplicate of this table, compressed with ALP instead
statement ok
PRAGMA force_compression='alprd'
statement ok
create table random_alp_double as select * from random_double;
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('random_alp_double') WHERE segment_type == 'double' AND compression != 'ALPRD';
----
# Assert that the data was not corrupted by compressing to ALP
query I sort r1
select * from random_double;
----
query I sort r1
select * from random_alp_double;
----

119
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_bitwidths.test_slow vendored Normal file
View File

@@ -0,0 +1,119 @@
# name: test/sql/storage/compression/bitpacking/bitpacking_bitwidths.test_slow
# description: Test bitpacking with values that compress to all different widths
# group: [bitpacking]
# load the DB from disk
load __TEST_DIR__/test_bitpacking.db
statement ok
PRAGMA force_compression = 'bitpacking'
foreach bitpacking_mode delta_for for constant_delta constant
foreach typesize 8 16 32 64
statement ok
PRAGMA force_bitpacking_mode='${bitpacking_mode}'
statement ok
CREATE TABLE test_unsigned AS SELECT cast(2**(i//2048) as UINT${typesize}) i FROM range(${typesize}*2048) tbl(i);
statement ok
CHECKPOINT;
query I
SELECT count(*)=${typesize} FROM (SELECT count(i) FROM test_unsigned GROUP BY i);
----
1
query I
SELECT DISTINCT(c_i) FROM (SELECT count(i) AS c_i FROM test_unsigned GROUP BY i);
----
2048
query IIII
select count(*)=1, count(*)=(${typesize}-1) , AVG(c_i), (i//delta) AS diff_to_next_row from (
SELECT i, count(i) as c_i, lag(i, 1) OVER (ORDER BY i) delta FROM test_unsigned GROUP BY i
) GROUP BY diff_to_next_row ORDER BY ALL;
----
False True 2048.000000 2
True False 2048.000000 NULL
statement ok
CREATE TABLE test_signed_neg AS SELECT cast(-(2**(i//2048)) as INT${typesize}) i FROM range(${typesize}*2048) tbl(i);
statement ok
CHECKPOINT;
query I
SELECT count(*)=${typesize} FROM (SELECT count(i) FROM test_signed_neg GROUP BY i);
----
1
query I
SELECT DISTINCT(c_i) FROM (SELECT count(i) AS c_i FROM test_signed_neg GROUP BY i);
----
2048
query IIII
select count(*)=1, count(*)=(${typesize}-1) , AVG(c_i), (i//delta) AS diff_to_next_row from (
SELECT i, count(i) as c_i, lag(i, 1) OVER (ORDER BY i) delta FROM test_unsigned GROUP BY i
) GROUP BY diff_to_next_row ORDER BY ALL;
----
False True 2048.000000 2
True False 2048.000000 NULL
statement ok
CREATE TABLE test_signed_pos AS SELECT cast(2**(i//2048) as INT${typesize}) i FROM range((${typesize}-1)*2048) tbl(i);
statement ok
CHECKPOINT;
query I
SELECT count(*)=(${typesize}-1) FROM (SELECT count(i) FROM test_signed_pos GROUP BY i);
----
1
query I
SELECT DISTINCT(c_i) FROM (SELECT count(i) AS c_i FROM test_signed_neg GROUP BY i);
----
2048
query IIII
select count(*)=1, count(*)=(${typesize}-1) , AVG(c_i), (i//delta) AS diff_to_next_row from (
SELECT i, count(i) as c_i, lag(i, 1) OVER (ORDER BY i) delta FROM test_unsigned GROUP BY i
) GROUP BY diff_to_next_row ORDER BY ALL;
----
False True 2048.000000 2
True False 2048.000000 NULL
statement ok
DROP TABLE test_unsigned
statement ok
DROP TABLE test_signed_neg
statement ok
DROP TABLE test_signed_pos
endloop
foreach type <integral> bool
statement ok
CREATE TABLE test_nullpack AS SELECT CAST((i//3000)%2 as ${type}) as i FROM range(0,12000) tbl(i);
statement ok
CHECKPOINT;
query I
SELECT AVG(cast (i as int)) FROM test_nullpack
----
0.5
statement ok
drop table test_nullpack
endloop
endloop

253
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_compression_ratio.test_slow vendored Normal file
View File

@@ -0,0 +1,253 @@
# name: test/sql/storage/compression/bitpacking/bitpacking_compression_ratio.test_slow
# description: Assert bitpacking compression ratio is within reasonable margins for each mode
# group: [bitpacking]
# This test defaults to another compression function for smaller block sizes,
# because the bitpacking groups no longer fit the blocks.
require block_size 262144
load __TEST_DIR__/test_bitpacking.db
#### CONSTANT MODE Compression ratio calculation:
# For single row group (note we choose values such that we don't create the constant segments):
# 59 vectors with CONSTANT mode, the last one will be FOR mode
# Total compressed bytes = 59*(8+4) + 1*(2048/8 + 8 + 8 + 4) = 984
# Total uncompressed bytes = 120000*8 = 960000
# Ratio ~= 975x
# However, because this completely fills up a block and we do not support block sharing yet, we waste a lot of space
statement ok
PRAGMA force_compression='bitpacking'
statement ok
PRAGMA force_bitpacking_mode='constant'
statement ok
CREATE TABLE test_bitpacked AS SELECT (i//119000::INT64)::INT64 AS i FROM range(0, 120000000) tbl(i);
statement ok
checkpoint
statement ok
PRAGMA force_compression='uncompressed'
statement ok
CREATE TABLE test_uncompressed AS SELECT i::INT64 FROM range(0, 120000000) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('test_bitpacked') WHERE segment_type != 'VALIDITY' AND compression != 'BitPacking';
----
query I
SELECT compression FROM pragma_storage_info('test_uncompressed') WHERE segment_type != 'VALIDITY' AND compression != 'Uncompressed';
----
query II
select (uncompressed::FLOAT / bitpacked::FLOAT) > 700, (uncompressed::FLOAT / bitpacked::FLOAT) < 1000 FROM (
select
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
)
----
True True
statement ok
drop table test_bitpacked;
drop table test_uncompressed;
#### CONSTANT DELTA MODE Compression ratio calculation:
# For single row group
# 60 vectors with a constant increase (1)
# Total compressed bytes = 60*(8+8+4) = 1200
# Total uncompressed bytes = 120000*8 = 960000
# Expected Ratio ~= 800x
statement ok
PRAGMA force_compression='bitpacking'
statement ok
PRAGMA force_bitpacking_mode='constant_delta'
statement ok
CREATE TABLE test_bitpacked AS SELECT i::INT64 AS i FROM range(0, 120000000) tbl(i);
statement ok
checkpoint
statement ok
PRAGMA force_compression='uncompressed'
statement ok
CREATE TABLE test_uncompressed AS SELECT i::INT64 AS i FROM range(0, 120000000) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('test_bitpacked') WHERE segment_type != 'VALIDITY' AND compression != 'BitPacking';
----
query I
SELECT compression FROM pragma_storage_info('test_uncompressed') WHERE segment_type != 'VALIDITY' AND compression != 'Uncompressed';
----
statement ok
checkpoint
query II
select (uncompressed::FLOAT / bitpacked::FLOAT) > 600, (uncompressed::FLOAT / bitpacked::FLOAT) < 800 FROM (
select
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
)
----
True True
statement ok
drop table test_bitpacked;
drop table test_uncompressed;
#### DELTA FOR MODE Compression ratio calculation:
# For single row group
# 60 vectors with DELTA_FOR mode smallest possible compression
# Total compressed bytes = 60*(8+8+4+(2048/8)) = 16560
# Total uncompressed bytes = 120000*8 = 960000
# Expected Ratio ~= 58x
statement ok
PRAGMA force_compression='bitpacking'
statement ok
PRAGMA force_bitpacking_mode='delta_for'
statement ok
CREATE TABLE test_bitpacked AS SELECT i//2::INT64 AS i FROM range(0, 120000000) tbl(i);
statement ok
checkpoint
statement ok
PRAGMA force_compression='uncompressed'
statement ok
CREATE TABLE test_uncompressed AS SELECT i AS i FROM range(0, 120000000) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('test_bitpacked') WHERE segment_type != 'VALIDITY' AND compression != 'BitPacking';
----
query I
SELECT compression FROM pragma_storage_info('test_uncompressed') WHERE segment_type != 'VALIDITY' AND compression != 'Uncompressed';
----
statement ok
checkpoint
query II
select (uncompressed::FLOAT / bitpacked::FLOAT) > 50, (uncompressed::FLOAT / bitpacked::FLOAT) < 60 FROM (
select
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
)
----
True True
statement ok
drop table test_bitpacked;
drop table test_uncompressed;
# FOR MODE Compression ratio calculation:
# For single row group
# 60 vectors with DELTA_FOR mode smallest possible compression
# Total compressed bytes = 60*(8+4+(2048/8)) = 16080
# Total uncompressed bytes = 120000*8 = 960000
# Expected Ratio ~= 60x
statement ok
PRAGMA force_compression='bitpacking'
statement ok
PRAGMA force_bitpacking_mode='for'
statement ok
CREATE TABLE test_bitpacked AS SELECT i%2::INT64 AS i FROM range(0, 120000000) tbl(i);
statement ok
checkpoint
statement ok
PRAGMA force_compression='uncompressed'
statement ok
CREATE TABLE test_uncompressed AS SELECT i::INT64 AS i FROM range(0, 120000000) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('test_bitpacked') WHERE segment_type != 'VALIDITY' AND compression != 'BitPacking';
----
query I
SELECT compression FROM pragma_storage_info('test_uncompressed') WHERE segment_type != 'VALIDITY' AND compression != 'Uncompressed';
----
statement ok
checkpoint
query II
select (uncompressed::FLOAT / bitpacked::FLOAT) > 50, (uncompressed::FLOAT / bitpacked::FLOAT) < 60 FROM (
select
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
)
----
True True
statement ok
drop table test_bitpacked;
drop table test_uncompressed;
statement ok
PRAGMA force_bitpacking_mode='none'
# Assert that all supported types do in fact compress
foreach type int8 int16 int32 int64 uint8 uint16 uint32 uint64 decimal(4,1) decimal(8,1) decimal(12,1) decimal(18,1) bool
statement ok
PRAGMA force_compression='uncompressed';
statement ok
CREATE TABLE test_uncompressed AS SELECT (i%2)::${type} FROM range(0, 2500000) tbl(i);
statement ok
checkpoint
statement ok
PRAGMA force_compression='bitpacking'
statement ok
CREATE TABLE test_bitpacked AS SELECT (i%2)::${type} FROM range(0, 2500000) tbl(i);
statement ok
checkpoint
# assert compression ratio >2 wich should be achieved for even the smallest types for this data
query II
select (uncompressed::FLOAT / bitpacked::FLOAT) > 2, CAST(1 as ${type}) FROM (
select
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
)
----
1 1
statement ok
drop table test_bitpacked
statement ok
drop table test_uncompressed
endloop

194
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_compression_ratio_hugeint.test_slow vendored Normal file
View File

@@ -0,0 +1,194 @@
# name: test/sql/storage/compression/bitpacking/bitpacking_compression_ratio_hugeint.test_slow
# description: Assert bitpacking compression ratio is within reasonable margins for each mode
# group: [bitpacking]
# This test defaults to another compression function for smaller block sizes,
# because the bitpacking groups no longer fit the blocks.
require block_size 262144
load __TEST_DIR__/test_bitpacking.db
#### CONSTANT MODE:
# Ratio ~= 1000x
statement ok
PRAGMA force_compression='bitpacking'
statement ok
PRAGMA force_bitpacking_mode='constant'
statement ok
CREATE TABLE test_bitpacked AS SELECT (i//119000::INT64)::HUGEINT AS i FROM range(0, 120000000) tbl(i);
statement ok
checkpoint
statement ok
PRAGMA force_compression='uncompressed'
statement ok
CREATE TABLE test_uncompressed AS SELECT i::HUGEINT FROM range(0, 120000000) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('test_bitpacked') WHERE segment_type != 'VALIDITY' AND compression != 'BitPacking';
----
query I
SELECT compression FROM pragma_storage_info('test_uncompressed') WHERE segment_type != 'VALIDITY' AND compression != 'Uncompressed';
----
query II
select (uncompressed::FLOAT / bitpacked::FLOAT) > 900, (uncompressed::FLOAT / bitpacked::FLOAT) < 1200 FROM (
select
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
)
----
True True
statement ok
drop table test_bitpacked;
drop table test_uncompressed;
#### CONSTANT DELTA MODE:
# Expected Ratio ~= 800x
statement ok
PRAGMA force_compression='bitpacking'
statement ok
PRAGMA force_bitpacking_mode='constant_delta'
statement ok
CREATE TABLE test_bitpacked AS SELECT i::HUGEINT AS i FROM range(0, 120000000) tbl(i);
statement ok
checkpoint
statement ok
PRAGMA force_compression='uncompressed'
statement ok
CREATE TABLE test_uncompressed AS SELECT i::HUGEINT AS i FROM range(0, 120000000) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('test_bitpacked') WHERE segment_type != 'VALIDITY' AND compression != 'BitPacking';
----
query I
SELECT compression FROM pragma_storage_info('test_uncompressed') WHERE segment_type != 'VALIDITY' AND compression != 'Uncompressed';
----
statement ok
checkpoint
query II
select (uncompressed::FLOAT / bitpacked::FLOAT) > 600, (uncompressed::FLOAT / bitpacked::FLOAT) < 800 FROM (
select
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
)
----
True True
statement ok
drop table test_bitpacked;
drop table test_uncompressed;
#### DELTA FOR MODE:
# Expected Ratio ~= 50x
statement ok
PRAGMA force_compression='bitpacking'
statement ok
PRAGMA force_bitpacking_mode='delta_for'
statement ok
CREATE TABLE test_bitpacked AS SELECT i//2::HUGEINT AS i FROM range(0, 120000000) tbl(i);
statement ok
checkpoint
statement ok
PRAGMA force_compression='uncompressed'
statement ok
CREATE TABLE test_uncompressed AS SELECT i AS i FROM range(0, 120000000) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('test_bitpacked') WHERE segment_type != 'VALIDITY' AND compression != 'BitPacking';
----
query I
SELECT compression FROM pragma_storage_info('test_uncompressed') WHERE segment_type != 'VALIDITY' AND compression != 'Uncompressed';
----
statement ok
checkpoint
query II
select (uncompressed::FLOAT / bitpacked::FLOAT) > 40, (uncompressed::FLOAT / bitpacked::FLOAT) < 60 FROM (
select
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
)
----
True True
statement ok
drop table test_bitpacked;
drop table test_uncompressed;
# FOR MODE:
# Expected Ratio ~= 95x
statement ok
PRAGMA force_compression='bitpacking'
statement ok
PRAGMA force_bitpacking_mode='for'
statement ok
CREATE TABLE test_bitpacked AS SELECT i%2::HUGEINT AS i FROM range(0, 120000000) tbl(i);
statement ok
checkpoint
statement ok
PRAGMA force_compression='uncompressed'
statement ok
CREATE TABLE test_uncompressed AS SELECT i::HUGEINT AS i FROM range(0, 120000000) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('test_bitpacked') WHERE segment_type != 'VALIDITY' AND compression != 'BitPacking';
----
query I
SELECT compression FROM pragma_storage_info('test_uncompressed') WHERE segment_type != 'VALIDITY' AND compression != 'Uncompressed';
----
statement ok
checkpoint
query II
select (uncompressed::FLOAT / bitpacked::FLOAT) > 90, (uncompressed::FLOAT / bitpacked::FLOAT) < 100 FROM (
select
(select count(distinct block_id) from pragma_storage_info('test_bitpacked') where segment_type not in('VARCHAR', 'VALIDITY')) as bitpacked,
(select count(distinct block_id) from pragma_storage_info('test_uncompressed') where segment_type not in('VARCHAR', 'VALIDITY')) as uncompressed
)
----
True True
statement ok
drop table test_bitpacked;
drop table test_uncompressed;

69
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_constant_delta.test vendored Normal file
View File

@@ -0,0 +1,69 @@
# name: test/sql/storage/compression/bitpacking/bitpacking_constant_delta.test
# description: Test that will use the BitpackingMode::CONSTANT_DELTA compression mode
# group: [bitpacking]
# This test defaults to another compression function for smaller block sizes,
# because the bitpacking groups no longer fit the blocks.
require block_size 262144
load __TEST_DIR__/test_bitpacking.db
statement ok
PRAGMA force_compression = 'bitpacking'
foreach bitpacking_mode delta_for for constant_delta constant
statement ok
PRAGMA force_bitpacking_mode='${bitpacking_mode}'
foreach type int8 int16 int32 int64 uint8 uint16 uint32 uint64 hugeint decimal(4,1) decimal(8,1) decimal(12,1) decimal(18,1)
statement ok
CREATE TABLE test (c ${type});
statement ok
INSERT INTO test SELECT 2+i*2::${type} FROM range(0,5) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('test') where segment_type != 'VALIDITY' and compression != 'BitPacking'
----
query I
SELECT * FROM test;
----
2
4
6
8
10
statement ok
DROP TABLE test
endloop
statement ok
CREATE TABLE test (c INT64);
statement ok
INSERT INTO test SELECT i from range(0,130000) tbl(i);
statement ok
checkpoint
query I
SELECT compression FROM pragma_storage_info('test') where segment_type != 'VALIDITY' and compression != 'BitPacking'
----
query I
SELECT avg(c) FROM test;
----
64999.5
statement ok
DROP TABLE test
endloop

68
external/duckdb/test/sql/storage/compression/bitpacking/bitpacking_delta.test_slow vendored Normal file
View File

@@ -0,0 +1,68 @@
# name: test/sql/storage/compression/bitpacking/bitpacking_delta.test_slow
# description: Test some large incompressible data
# group: [bitpacking]
# This test defaults to another compression function for smaller block sizes,
# because the bitpacking groups no longer fit the blocks.
require block_size 262144
load __TEST_DIR__/bitpacking_uncompressible.db
statement ok
PRAGMA force_compression='bitpacking'
foreach bitpacking_mode delta_for for constant_delta constant
statement ok
PRAGMA force_bitpacking_mode='${bitpacking_mode}'
# simple compression with few values
statement ok
CREATE TABLE test_delta_full_range (a UINT64);
# Insert multiple ranges so that each method can be used on at least on the the ranges
statement ok
INSERT INTO test_delta_full_range select case when i%2=0 then 0 else 18446744073709551615 end from range(0,1000000) tbl(i);
query II
select a, count(*) from test_delta_full_range group by a order by a;
----
0 500000
18446744073709551615 500000
query I
SELECT DISTINCT compression FROM pragma_storage_info('test_delta_full_range') where segment_type = 'UBIGINT'
----
BitPacking
statement ok
drop table test_delta_full_range
endloop
# Do the same thing and confirm we don't bitpack here
statement ok
PRAGMA force_compression='none'
# simple compression with few values
statement ok
CREATE TABLE test_delta_full_range (a UINT64);
# Insert multiple ranges so that each method can be used on at least on the the ranges
statement ok
INSERT INTO test_delta_full_range select case when i%2=0 then 0 else 18446744073709551615 end from range(0,1000000) tbl(i);
query II
select a, count(*) from test_delta_full_range group by a order by a;
----
0 500000
18446744073709551615 500000
query I
SELECT DISTINCT compression FROM pragma_storage_info('test_delta_full_range') where segment_type = 'UBIGINT'
----
Uncompressed
statement ok
drop table test_delta_full_range

Some files were not shown because too many files have changed in this diff Show More