should be it

external/duckdb/test/persistence/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_library_unity(test_persistence OBJECT test_file_matches_wal.cpp
+                  test_locking.cpp test_persistence.cpp test_sequence_crash.cpp)
+set(ALL_OBJECT_FILES
+    ${ALL_OBJECT_FILES} $<TARGET_OBJECTS:test_persistence>
+    PARENT_SCOPE)

external/duckdb/test/persistence/test_file_matches_wal.cpp

@@ -0,0 +1,81 @@
+#include "catch.hpp"
+#include "test_helpers.hpp"
+
+#include <fstream>
+#include <iostream>
+
+using namespace duckdb;
+using namespace std;
+
+TEST_CASE("Test replaying mismatching WAL files", "[persistence][.]") {
+	duckdb::unique_ptr<QueryResult> result;
+	DuckDB db(nullptr);
+	Connection con(db);
+
+	// Configuration to avoid checkpointing.
+	REQUIRE_NO_FAIL(con.Query("PRAGMA disable_checkpoint_on_shutdown;"));
+	REQUIRE_NO_FAIL(con.Query("SET checkpoint_threshold = '10.0 GB';"));
+	REQUIRE_NO_FAIL(con.Query("SET storage_compatibility_version='v1.4.0';"));
+	REQUIRE_NO_FAIL(con.Query("SET threads = 1;"));
+
+	auto test_dir = TestDirectoryPath();
+	string db_name = "my_db";
+	string db_path = test_dir + "/" + db_name + ".db";
+	string wal_path = db_path + ".wal";
+
+	// Create initial file and WAL.
+	REQUIRE_NO_FAIL(con.Query("ATTACH '" + db_path + "';"));
+	REQUIRE_NO_FAIL(con.Query("CREATE TABLE my_db.tbl AS SELECT range AS id FROM range(100);"));
+	REQUIRE_NO_FAIL(con.Query("DETACH my_db;"));
+
+	// Make a copy of the file and the WAL file.
+	string too_new_path_file = test_dir + "/" + db_name + "_too_new.db";
+	string too_old_path_wal = test_dir + "/" + db_name + "_too_old.db.wal";
+
+	string copy_file_cmd = "cp " + db_path + " " + too_new_path_file;
+	REQUIRE(system(copy_file_cmd.c_str()) == 0);
+	string copy_wal_cmd = "cp " + wal_path + " " + too_old_path_wal;
+	REQUIRE(system(copy_wal_cmd.c_str()) == 0);
+
+	// Replay the WAL and make more changes.
+	REQUIRE_NO_FAIL(con.Query("ATTACH '" + db_path + "';"));
+	REQUIRE_NO_FAIL(con.Query("CHECKPOINT my_db;"));
+	REQUIRE_NO_FAIL(con.Query("INSERT INTO my_db.tbl SELECT range + 400 FROM range(100);"));
+	REQUIRE_NO_FAIL(con.Query("DETACH my_db;"));
+
+	// Make a copy of the file and the WAL file.
+	string too_old_path_file = test_dir + "/" + db_name + "_too_old.db";
+	string too_new_path_wal = test_dir + "/" + db_name + "_too_new.db.wal";
+
+	copy_file_cmd = "cp " + db_path + " " + too_old_path_file;
+	REQUIRE(system(copy_file_cmd.c_str()) == 0);
+	copy_wal_cmd = "cp " + wal_path + " " + too_new_path_wal;
+	REQUIRE(system(copy_wal_cmd.c_str()) == 0);
+
+	result = con.Query("ATTACH '" + too_old_path_file + "';");
+	REQUIRE(result->HasError());
+	string error_msg = result->GetError();
+	REQUIRE(StringUtil::Contains(error_msg, "older"));
+
+	result = con.Query("ATTACH '" + too_new_path_file + "';");
+	REQUIRE(result->HasError());
+	error_msg = result->GetError();
+	REQUIRE(StringUtil::Contains(error_msg, "newer"));
+
+	// Create and initialize a different file.
+	string other_db_path = test_dir + "/my_other_db.db";
+	REQUIRE_NO_FAIL(con.Query("ATTACH '" + other_db_path + "';"));
+	REQUIRE_NO_FAIL(con.Query("CREATE TABLE my_other_db.tbl AS SELECT range AS id FROM range(100);"));
+	REQUIRE_NO_FAIL(con.Query("CHECKPOINT my_other_db;"));
+	REQUIRE_NO_FAIL(con.Query("DETACH my_other_db;"));
+
+	// Also copy this WAL for the file mismatch test.
+	string other_path_wal = test_dir + "/my_other_db.db.wal";
+	copy_wal_cmd = "cp " + wal_path + " " + other_path_wal;
+	REQUIRE(system(copy_wal_cmd.c_str()) == 0);
+
+	result = con.Query("ATTACH '" + other_db_path + "';");
+	REQUIRE(result->HasError());
+	error_msg = result->GetError();
+	REQUIRE(StringUtil::Contains(error_msg, "WAL does not match database file"));
+}

external/duckdb/test/persistence/test_locking.cpp

@@ -0,0 +1,110 @@
+#include "catch.hpp"
+#include "duckdb/common/file_system.hpp"
+#include "duckdb.hpp"
+#include "test_helpers.hpp"
+
+#include <signal.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#ifdef __MVS__
+#define MAP_ANONYMOUS 0x0
+#endif
+
+using namespace duckdb;
+using namespace std;
+
+#define BOOL_COUNT 3
+
+TEST_CASE("Test write lock with multiple processes", "[persistence][.]") {
+	uint64_t *count =
+	    (uint64_t *)mmap(NULL, sizeof(uint64_t), PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_SHARED, 0, 0);
+	*count = 0;
+
+	string dbdir = TestCreatePath("writelocktest");
+	DeleteDatabase(dbdir);
+	// test write lock
+	// fork away a child
+	pid_t pid = fork();
+	if (pid == 0) {
+		// child process
+		// open db for writing
+		DuckDB db(dbdir);
+		Connection con(db);
+		// opened db for writing
+		// insert some values
+		(*count)++;
+		con.Query("CREATE TABLE a(i INTEGER)");
+		con.Query("INSERT INTO a VALUES(42)");
+		while (true) {
+			con.Query("SELECT * FROM a");
+			usleep(100);
+		}
+	} else if (pid > 0) {
+		duckdb::unique_ptr<DuckDB> db;
+		// parent process
+		// sleep a bit to wait for child process
+		while (*count == 0) {
+			usleep(100);
+		}
+		// try to open db for writing, this should fail
+		REQUIRE_THROWS(db = make_uniq<DuckDB>(dbdir));
+		// kill the child
+		if (kill(pid, SIGKILL) != 0) {
+			FAIL();
+		}
+	}
+}
+
+TEST_CASE("Test read lock with multiple processes", "[persistence][.]") {
+	uint64_t *count =
+	    (uint64_t *)mmap(NULL, sizeof(uint64_t), PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_SHARED, 0, 0);
+	*count = 0;
+
+	string dbdir = TestCreatePath("readlocktest");
+	DeleteDatabase(dbdir);
+
+	// create the database
+	{
+		DuckDB db(dbdir);
+		Connection con(db);
+		REQUIRE_NO_FAIL(con.Query("CREATE TABLE a(i INTEGER)"));
+		REQUIRE_NO_FAIL(con.Query("INSERT INTO a VALUES (42)"));
+	}
+	// test read lock
+	pid_t pid = fork();
+	DBConfig config;
+	config.options.access_mode = AccessMode::READ_ONLY;
+	if (pid == 0) {
+		// child process
+		// open db for reading
+		DuckDB db(dbdir, &config);
+		Connection con(db);
+
+		(*count)++;
+		// query some values
+		con.Query("SELECT i+2 FROM a");
+		while (true) {
+			usleep(100);
+			con.Query("SELECT * FROM a");
+		}
+	} else if (pid > 0) {
+		duckdb::unique_ptr<DuckDB> db;
+		// parent process
+		// sleep a bit to wait for child process
+		while (*count == 0) {
+			usleep(100);
+		}
+		// try to open db for writing, this should fail
+		REQUIRE_THROWS(db = make_uniq<DuckDB>(dbdir));
+		// but opening db for reading should work
+		REQUIRE_NOTHROW(db = make_uniq<DuckDB>(dbdir, &config));
+		// we can query the database
+		Connection con(*db);
+		REQUIRE_NO_FAIL(con.Query("SELECT * FROM a"));
+		// kill the child
+		if (kill(pid, SIGKILL) != 0) {
+			FAIL();
+		}
+	}
+}

external/duckdb/test/persistence/test_persistence.cpp

@@ -0,0 +1,74 @@
+#include "catch.hpp"
+#include "duckdb/common/file_system.hpp"
+#include "duckdb.hpp"
+#include "test_helpers.hpp"
+
+#include <signal.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#ifdef __MVS__
+#define MAP_ANONYMOUS 0x0
+#endif
+
+using namespace duckdb;
+using namespace std;
+
+TEST_CASE("Test transactional integrity when facing process aborts", "[persistence][.]") {
+	duckdb::unique_ptr<FileSystem> fs = FileSystem::CreateLocal();
+
+	// shared memory to keep track of insertions
+	size_t *count = (size_t *)mmap(NULL, sizeof(size_t), PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_SHARED, 0, 0);
+
+	string db_folder_parent = TestCreatePath("llstorage");
+	if (fs->DirectoryExists(db_folder_parent)) {
+		fs->RemoveDirectory(db_folder_parent);
+	}
+	fs->CreateDirectory(db_folder_parent);
+
+	string db_folder = fs->JoinPath(db_folder_parent, "dbfolder");
+	{
+		DuckDB db(db_folder);
+		Connection con(db);
+		con.Query("CREATE TABLE a (i INTEGER)");
+	}
+
+	// fork away a child to be mercilessy shot in a bit
+	pid_t pid = fork();
+
+	if (pid == 0) { // child process
+		DuckDB db(db_folder);
+		Connection con(db);
+		while (true) {
+			con.Query("INSERT INTO a VALUES(42)");
+			(*count)++;
+		}
+	} else if (pid > 0) { // parent process
+		// wait until child has inserted at least 1000 rows
+		while (*count < 1000) {
+			usleep(100);
+		}
+		if (kill(pid, SIGKILL) != 0) {
+			FAIL();
+		}
+		duckdb::unique_ptr<DuckDB> db;
+		// it may take some time for the OS to reclaim the lock
+		// loop and wait until the database is successfully started again
+		for (size_t i = 0; i < 1000; i++) {
+			usleep(10000);
+			try {
+				db = make_uniq<DuckDB>(db_folder);
+			} catch (...) {
+			}
+			if (db) {
+				break;
+			}
+		}
+		Connection con(*db);
+		auto res = con.Query("SELECT COUNT(*) FROM a");
+		// there may be an off-by-one if we kill exactly between query and count increment
+		REQUIRE(std::abs((int64_t)(res->GetValue(0, 0).GetValue<int64_t>() - *count)) < 2);
+	} else {
+		FAIL();
+	}
+}

external/duckdb/test/persistence/test_sequence_crash.cpp

@@ -0,0 +1,128 @@
+#include "catch.hpp"
+#include "duckdb.hpp"
+#include "test_helpers.hpp"
+
+#include <signal.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <thread>
+
+using namespace duckdb;
+using namespace std;
+
+TEST_CASE("Test that sequence never returns the same value twice even with aborts", "[persistence][.]") {
+	// disabled test for now
+	return;
+
+	string dbdir = TestCreatePath("defaultseq");
+	DeleteDatabase(dbdir);
+	// create a database
+	{
+		DuckDB db(dbdir);
+		Connection con(db);
+		REQUIRE_NO_FAIL(con.Query("CREATE SEQUENCE seq"));
+		REQUIRE_NO_FAIL(con.Query("CREATE TABLE a (i INTEGER DEFAULT nextval('seq'), j INTEGER)"));
+	}
+
+	// now fork the process a bunch of times
+	for (int i = 0; i < 100; i++) {
+		// fork the process
+		pid_t pid = fork();
+		if (pid == 0) {
+			// child process, connect to the database and start inserting values
+			DuckDB db(dbdir);
+			Connection con(db);
+			while (true) {
+				con.Query("INSERT INTO a (j) VALUES(1)");
+			}
+		} else if (pid > 0) {
+			// parent process, sleep a bit
+			usleep(100000);
+			// send SIGKILL to the child
+			if (kill(pid, SIGKILL) != 0) {
+				FAIL();
+			}
+			// sleep a bit before continuing the loop, to make sure the lock on the database was released
+			usleep(100000);
+		} else {
+			FAIL();
+		}
+	}
+	// now connect to the database
+	{
+		DuckDB db(dbdir);
+		Connection con(db);
+		// verify that "i" only has unique values from the sequence
+		// i.e. COUNT = COUNT(DISTINCT)
+		auto result = con.Query("SELECT COUNT(i), COUNT(DISTINCT i) FROM a");
+		REQUIRE(CHECK_COLUMN(result, 1, {result->GetValue(0, 0)}));
+	}
+	DeleteDatabase(dbdir);
+}
+
+static void write_entries_to_table(DuckDB *db, int i) {
+	Connection con(*db);
+	if (i % 2 == 0) {
+		// i % 2 = 0, insert values
+		while (true) {
+			con.Query("INSERT INTO a (j) VALUES(1)");
+		}
+	} else {
+		// use nextval in select clause
+		while (true) {
+			con.Query("SELECT nextval('seq')");
+		}
+	}
+}
+
+TEST_CASE("Test that sequence never returns the same value twice even with aborts and concurrent usage",
+          "[persistence][.]") {
+	// disabled test for now
+	return;
+
+	string dbdir = TestCreatePath("defaultseqconcurrent");
+	DeleteDatabase(dbdir);
+	// create a database
+	{
+		DuckDB db(dbdir);
+		Connection con(db);
+		REQUIRE_NO_FAIL(con.Query("CREATE SEQUENCE seq"));
+		REQUIRE_NO_FAIL(con.Query("CREATE TABLE a (i INTEGER DEFAULT nextval('seq'), j INTEGER)"));
+	}
+	// now fork the process a bunch of times
+	for (int i = 0; i < 100; i++) {
+		// fork the process
+		pid_t pid = fork();
+		if (pid == 0) {
+			// child process, connect to the database and start inserting values in separate threads
+			DuckDB db(dbdir);
+			thread write_threads[8];
+			for (size_t i = 0; i < 8; i++) {
+				write_threads[i] = thread(write_entries_to_table, &db, i);
+			}
+			while (true)
+				;
+		} else if (pid > 0) {
+			// parent process, sleep a bit
+			usleep(100000);
+			// send SIGKILL to the child
+			if (kill(pid, SIGKILL) != 0) {
+				FAIL();
+			}
+			// sleep a bit before continuing the loop, to make sure the lock on the database was released
+			usleep(100000);
+		} else {
+			FAIL();
+		}
+	}
+	// now connect to the database
+	{
+		DuckDB db(dbdir);
+		Connection con(db);
+		// verify that "i" only has unique values from the sequence
+		// i.e. COUNT = COUNT(DISTINCT)
+		auto result = con.Query("SELECT COUNT(i), COUNT(DISTINCT i) FROM a");
+		REQUIRE(CHECK_COLUMN(result, 1, {result->GetValue(0, 0)}));
+	}
+	DeleteDatabase(dbdir);
+}