should be it

external/duckdb/test/sql/storage/reclaim_space/reclaim_space_drop_column_overflow_strings.test_slow

@@ -0,0 +1,93 @@
+# name: test/sql/storage/reclaim_space/reclaim_space_drop_column_overflow_strings.test_slow
+# description: Test that we reclaim space when dropping columns containing overflow strings
+# group: [reclaim_space]
+
+load __TEST_DIR__/reclaim_space_drop_column_overflow_strings.db
+
+statement ok
+PRAGMA force_compression='Uncompressed';
+
+statement ok
+PRAGMA force_checkpoint;
+
+statement ok
+CREATE TABLE strings AS SELECT i, repeat('X', case when i%17=0 then 5000 else i%7 end) AS s FROM generate_series(0,150000) tbl(i);
+
+statement ok
+CHECKPOINT;
+
+statement ok
+CHECKPOINT;
+
+query IIIIII
+SELECT AVG(STRLEN(s)), MIN(STRLEN(S)), MAX(STRLEN(S)), SUM(STRLEN(S)), MIN(S[1]), MAX(S[1]) FROM strings
+----
+296.955	0	5000	44543527	(empty)	X
+
+# For smaller block sizes (16KB) the total blocks alternate between a few values in the loop,
+# therefore, we need to compare to a range of total block counts.
+statement ok
+CREATE TABLE total_blocks_tbl AS SELECT total_blocks FROM pragma_database_size();
+
+statement ok
+create type test_result as UNION(
+	ok BOOL,
+	err STRUCT(
+		old BIGINT,
+		allowed_max DECIMAL(21,1),
+		actual BIGINT
+	)
+);
+
+loop i 0 30
+
+statement ok
+ALTER TABLE strings DROP COLUMN s;
+
+statement ok
+ALTER TABLE strings ADD COLUMN s VARCHAR;
+
+statement ok
+UPDATE strings SET s=repeat('X', case when i%17=0 then 5000 else i%7 end);
+
+query IIIIII
+SELECT AVG(STRLEN(s)), MIN(STRLEN(S)), MAX(STRLEN(S)), SUM(STRLEN(S)), MIN(S[1]), MAX(S[1]) FROM strings
+----
+296.955	0	5000	44543527	(empty)	X
+
+statement ok
+CHECKPOINT;
+
+# Ensure that the total blocks don't exceed the total blocks by more than 1.2.
+
+query I
+SELECT
+	CASE WHEN ${i} < 10
+		THEN True::test_result
+	WHEN current.total_blocks <= total_blocks_tbl.total_blocks * 1.2
+		THEN True::test_result
+	ELSE {
+			'old': total_blocks_tbl.total_blocks,
+			'allowed_max': total_blocks_tbl.total_blocks * 1.2,
+			'actual': current.total_blocks
+		}::test_result
+	END
+FROM pragma_database_size() AS current, total_blocks_tbl;
+----
+true
+
+# Adjust total_blocks_tbl to the count after 10 warm-up iterations.
+
+statement ok
+UPDATE total_blocks_tbl SET total_blocks = (
+	SELECT CASE WHEN ${i} < 10 THEN (SELECT current.total_blocks FROM pragma_database_size() AS current)
+		ELSE (total_blocks) END);
+
+restart
+
+query IIIIII
+SELECT AVG(STRLEN(s)), MIN(STRLEN(S)), MAX(STRLEN(S)), SUM(STRLEN(S)), MIN(S[1]), MAX(S[1]) FROM strings
+----
+296.955	0	5000	44543527	(empty)	X
+
+endloop

external/duckdb/test/sql/storage/reclaim_space/reclaim_space_drop_overflow_strings.test_slow

@@ -0,0 +1,130 @@
+# name: test/sql/storage/reclaim_space/reclaim_space_drop_overflow_strings.test_slow
+# description: Test that we reclaim space when dropping tables containing overflow strings
+# group: [reclaim_space]
+
+load __TEST_DIR__/reclaim_space_overflow_strings.db
+
+statement ok
+PRAGMA force_compression='Uncompressed';
+
+statement ok
+PRAGMA force_checkpoint;
+
+# Every 17th value has length 5000
+# Other values are between 0-6 in length
+statement ok
+CREATE TABLE strings AS SELECT
+	repeat('X', case when i%17=0 then 5000 else i%7 end) AS s
+FROM generate_series(0,150000) tbl(i);
+
+statement ok
+CHECKPOINT;
+
+statement ok
+CHECKPOINT;
+
+query IIIIII
+SELECT
+	AVG(STRLEN(s)),
+	MIN(STRLEN(S)),
+	MAX(STRLEN(S)),
+	SUM(STRLEN(S)),
+	MIN(S[1]),
+	MAX(S[1])
+FROM strings
+----
+296.955	0	5000	44543527	(empty)	X
+
+# for smaller block sizes (16KB) the total blocks alternate between a few values in the loop,
+# therefore, we need to compare to a range of total block counts
+statement ok
+CREATE TABLE total_blocks_tbl AS SELECT
+	total_blocks
+FROM pragma_database_size();
+
+statement ok
+create type test_result as UNION(
+	ok BOOL,
+	err STRUCT(
+		old BIGINT,
+		allowed_max DECIMAL(21,1),
+		actual BIGINT
+	)
+);
+
+loop i 0 10
+
+statement ok
+DROP TABLE strings;
+
+# Recreate the table
+statement ok
+CREATE TABLE strings AS SELECT
+	repeat('X', case when i%17=0 then 5000 else i%7 end) AS s
+FROM generate_series(0,150000) tbl(i);
+
+query IIIIII
+SELECT
+	AVG(STRLEN(s)),
+	MIN(STRLEN(S)),
+	MAX(STRLEN(S)),
+	SUM(STRLEN(S)),
+	MIN(S[1]),
+	MAX(S[1])
+FROM strings
+----
+296.955	0	5000	44543527	(empty)	X
+
+statement ok
+CHECKPOINT;
+
+# ensure that the total blocks don't exceed the total blocks after the first iteration
+# by more than 1.2
+
+query I
+SELECT
+	CASE WHEN ${i} = 0
+		THEN True::test_result
+	WHEN current.total_blocks <= total_blocks_tbl.total_blocks * 1.2
+		THEN True::test_result
+	ELSE {
+			'old': total_blocks_tbl.total_blocks,
+			'allowed_max': total_blocks_tbl.total_blocks * 1.2,
+			'actual': current.total_blocks
+		}::test_result
+	END
+FROM pragma_database_size() AS current, total_blocks_tbl;
+----
+true
+
+# adjust total_blocks_tbl once to the count after the first iteration
+
+statement ok
+UPDATE total_blocks_tbl SET total_blocks = (
+	SELECT
+		CASE WHEN ${i} = 0
+			THEN (SELECT current.total_blocks FROM pragma_database_size() AS current)
+		ELSE
+			(total_blocks)
+		END
+	);
+
+statement ok
+CHECKPOINT;
+
+restart
+
+query IIIIII
+SELECT
+	AVG(STRLEN(s)),
+	MIN(STRLEN(S)),
+	MAX(STRLEN(S)),
+	SUM(STRLEN(S)),
+	MIN(S[1]),
+	MAX(S[1])
+FROM strings
+----
+296.955	0	5000	44543527	(empty)	X
+
+# i
+endloop

external/duckdb/test/sql/storage/reclaim_space/reclaim_space_lists.test_slow

@@ -0,0 +1,52 @@
+# name: test/sql/storage/reclaim_space/reclaim_space_lists.test_slow
+# description: Test that we reclaim space of LIST tables
+# group: [reclaim_space]
+
+load __TEST_DIR__/test_reclaim_space.db
+
+statement ok
+PRAGMA force_checkpoint;
+
+statement ok
+SET force_compression='uncompressed';
+
+statement ok
+CREATE TABLE lists AS SELECT [i] l FROM range(10000000) tbl(i);
+
+statement ok
+CHECKPOINT;
+
+statement ok
+CHECKPOINT;
+
+query III
+SELECT MIN(l[1]), MAX(l[1]), COUNT(*) FROM lists;
+----
+0	9999999	10000000
+
+statement ok
+DROP TABLE lists;
+
+loop i 0 10
+
+statement ok
+CREATE TABLE lists${i} AS SELECT [i] l FROM range(10000000) tbl(i);
+
+query III
+SELECT MIN(l[1]), MAX(l[1]), COUNT(*) FROM lists${i};
+----
+0	9999999	10000000
+
+statement ok
+CHECKPOINT;
+
+statement ok
+DROP TABLE lists${i};
+
+statement ok
+CHECKPOINT;
+
+query I nosort expected_blocks
+SELECT round(total_blocks / 100.0) FROM pragma_database_size();
+
+endloop

external/duckdb/test/sql/storage/reclaim_space/test_reclaim_space_alter_type.test_slow

@@ -0,0 +1,78 @@
+# name: test/sql/storage/reclaim_space/test_reclaim_space_alter_type.test_slow
+# description: Test that we reclaim space when altering the type of a column
+# group: [reclaim_space]
+
+load __TEST_DIR__/test_reclaim_space.db
+
+statement ok
+PRAGMA force_checkpoint;
+
+statement ok
+CREATE TABLE integers AS SELECT i FROM range(1000000) tbl(i);
+
+statement ok
+CHECKPOINT;
+
+statement ok
+CHECKPOINT;
+
+query III
+SELECT MIN(i), MAX(i), COUNT(*) FROM integers
+----
+0	999999	1000000
+
+# For smaller block sizes (16KB) the total blocks alternate between a few values in the loop,
+# therefore, we need to compare to a range of total block counts.
+statement ok
+CREATE TABLE total_blocks_tbl AS SELECT total_blocks FROM pragma_database_size();
+
+loop i 0 20
+
+statement ok
+ALTER TABLE integers ALTER i SET DATA TYPE BIGINT;
+
+statement ok
+CHECKPOINT;
+
+query III
+SELECT MIN(i), MAX(i), COUNT(*) FROM integers
+----
+0	999999	1000000
+
+statement ok
+ALTER TABLE integers ALTER i SET DATA TYPE INTEGER;
+
+statement ok
+CHECKPOINT;
+
+query III
+SELECT MIN(i), MAX(i), COUNT(*) FROM integers
+----
+0	999999	1000000
+
+statement ok
+CHECKPOINT;
+
+# Ensure that the total blocks don't exceed the total blocks by more than 1.2.
+
+query I
+SELECT CASE WHEN ${i} < 10 THEN True
+	WHEN current.total_blocks <= total_blocks_tbl.total_blocks * 1.2 THEN True
+	ELSE False END
+FROM pragma_database_size() AS current, total_blocks_tbl;
+----
+1
+
+# Adjust total_blocks_tbl to the count after 10 warm-up iterations.
+
+statement ok
+UPDATE total_blocks_tbl SET total_blocks = (
+	SELECT CASE WHEN ${i} < 10 THEN (SELECT current.total_blocks FROM pragma_database_size() AS current)
+		ELSE (total_blocks) END);
+
+query III
+SELECT MIN(i), MAX(i), COUNT(*) FROM integers
+----
+0	999999	1000000
+
+endloop

external/duckdb/test/sql/storage/reclaim_space/test_reclaim_space_drop.test_slow

@@ -0,0 +1,48 @@
+# name: test/sql/storage/reclaim_space/test_reclaim_space_drop.test_slow
+# description: Test that we reclaim space when dropping tables
+# group: [reclaim_space]
+
+load __TEST_DIR__/test_reclaim_space.db
+
+statement ok
+PRAGMA force_checkpoint;
+
+statement ok
+CREATE TABLE integers AS SELECT i FROM range(10000000) tbl(i);
+
+statement ok
+CHECKPOINT;
+
+statement ok
+CHECKPOINT;
+
+query III
+SELECT MIN(i), MAX(i), COUNT(*) FROM integers
+----
+0	9999999	10000000
+
+loop i 0 10
+
+statement ok
+DROP TABLE integers;
+
+statement ok
+CREATE TABLE integers AS SELECT i FROM range(10000000) tbl(i);
+
+query III
+SELECT MIN(i), MAX(i), COUNT(*) FROM integers
+----
+0	9999999	10000000
+
+statement ok
+CHECKPOINT;
+
+query I nosort expected_blocks
+select round(total_blocks / 100.0) from pragma_database_size();
+
+query III
+SELECT MIN(i), MAX(i), COUNT(*) FROM integers
+----
+0	9999999	10000000
+
+endloop

external/duckdb/test/sql/storage/reclaim_space/test_reclaim_space_drop_column.test_slow

@@ -0,0 +1,71 @@
+# name: test/sql/storage/reclaim_space/test_reclaim_space_drop_column.test_slow
+# description: Test that we reclaim space when dropping and adding columns
+# group: [reclaim_space]
+
+load __TEST_DIR__/test_reclaim_space_drop_column.db
+
+statement ok
+PRAGMA force_checkpoint;
+
+statement ok
+CREATE TABLE integers AS SELECT i, i j FROM range(1000000) tbl(i);
+
+statement ok
+CHECKPOINT;
+
+statement ok
+CHECKPOINT;
+
+query III
+SELECT MIN(j), MAX(j), COUNT(*) FROM integers
+----
+0	999999	1000000
+
+# for smaller block sizes (16KB) the total blocks alternate between a few values in the loop,
+# therefore, we need to compare to a range of total block counts
+statement ok
+CREATE TABLE total_blocks_tbl AS SELECT total_blocks FROM pragma_database_size();
+
+loop i 0 10
+
+statement ok
+ALTER TABLE integers DROP COLUMN j;
+
+statement ok
+ALTER TABLE integers ADD COLUMN j INTEGER;
+
+statement ok
+UPDATE integers SET j=i;
+
+query III
+SELECT MIN(j), MAX(j), COUNT(*) FROM integers
+----
+0	999999	1000000
+
+statement ok
+CHECKPOINT;
+
+# ensure that the total blocks don't exceed the total blocks after the first iteration
+# by more than 1.2
+
+query I
+SELECT CASE WHEN ${i} = 0 THEN True
+	WHEN current.total_blocks <= total_blocks_tbl.total_blocks * 1.2 THEN True
+	ELSE False END
+FROM pragma_database_size() AS current, total_blocks_tbl;
+----
+1
+
+# adjust total_blocks_tbl once to the count after the first iteration
+
+statement ok
+UPDATE total_blocks_tbl SET total_blocks = (
+	SELECT CASE WHEN ${i} = 0 THEN (SELECT current.total_blocks FROM pragma_database_size() AS current)
+		ELSE (total_blocks) END);
+
+query III
+SELECT MIN(i), MAX(i), COUNT(*) FROM integers
+----
+0	999999	1000000
+
+endloop

external/duckdb/test/sql/storage/reclaim_space/test_reclaim_space_update.test_slow

@@ -0,0 +1,78 @@
+# name: test/sql/storage/reclaim_space/test_reclaim_space_update.test_slow
+# description: Test that we reclaim space when updating the values of a column
+# group: [reclaim_space]
+
+load __TEST_DIR__/test_reclaim_space.db
+
+statement ok
+PRAGMA force_checkpoint;
+
+statement ok
+CREATE TABLE integers AS SELECT i FROM range(1000000) tbl(i);
+
+statement ok
+CHECKPOINT;
+
+statement ok
+CHECKPOINT;
+
+query III
+SELECT MIN(i), MAX(i), COUNT(*) FROM integers
+----
+0	999999	1000000
+
+# For smaller block sizes (16KB) the total blocks alternate between a few values in the loop,
+# therefore, we need to compare to a range of total block counts.
+statement ok
+CREATE TABLE total_blocks_tbl AS SELECT total_blocks FROM pragma_database_size();
+
+loop i 0 20
+
+statement ok
+UPDATE integers SET i=i;
+
+statement ok
+CHECKPOINT;
+
+query III
+SELECT MIN(i), MAX(i), COUNT(*) FROM integers
+----
+0	999999	1000000
+
+statement ok
+UPDATE integers SET i=i;
+
+statement ok
+CHECKPOINT;
+
+query III
+SELECT MIN(i), MAX(i), COUNT(*) FROM integers
+----
+0	999999	1000000
+
+statement ok
+CHECKPOINT;
+
+# Ensure that the total blocks don't exceed the total blocks by more than 1.2.
+
+query I
+SELECT CASE WHEN ${i} < 10 THEN True
+	WHEN current.total_blocks <= total_blocks_tbl.total_blocks * 1.2 THEN True
+	ELSE False END
+FROM pragma_database_size() AS current, total_blocks_tbl;
+----
+1
+
+# Adjust total_blocks_tbl to the count after 10 warm-up iterations.
+
+statement ok
+UPDATE total_blocks_tbl SET total_blocks = (
+ 	SELECT CASE WHEN ${i} < 10 THEN (SELECT current.total_blocks FROM pragma_database_size() AS current)
+ 		ELSE (total_blocks) END);
+
+query III
+SELECT MIN(i), MAX(i), COUNT(*) FROM integers
+----
+0	999999	1000000
+
+endloop

external/duckdb/test/sql/storage/reclaim_space/test_reclaim_space_update_large_string.test

@@ -0,0 +1,57 @@
+# name: test/sql/storage/reclaim_space/test_reclaim_space_update_large_string.test
+# description: Test that we reclaim space when updating a large string
+# group: [reclaim_space]
+
+load __TEST_DIR__/test_reclaim_space.db
+
+statement ok
+PRAGMA force_checkpoint;
+
+# FIXME: overflow strings are not reclaimed yet when overwritten on updates
+mode skip
+
+# create a big string
+statement ok
+CREATE TABLE test (a VARCHAR);
+
+# length 1000000
+statement ok
+INSERT INTO test VALUES (repeat('a', 1000000));
+
+query I
+SELECT LENGTH(SUBSTRING(a, 0, 1000000)) FROM test
+----
+999999
+
+# checkpoint the string
+statement ok
+CHECKPOINT
+
+query I
+SELECT LENGTH(SUBSTRING(a, 0, 1000000)) FROM test
+----
+999999
+
+# now update by adding one character to the string on every iteration
+loop i 0 10
+
+statement ok
+UPDATE test SET a=concat(a, 'a')
+
+query I
+SELECT LENGTH(SUBSTRING(a, 0, 1000000)) FROM test
+----
+999999
+
+statement ok
+CHECKPOINT
+
+query I nosort expected_blocks
+select total_blocks from pragma_database_size();
+
+query I
+SELECT LENGTH(SUBSTRING(a, 0, 1000000)) FROM test
+----
+999999
+
+endloop