should be it

external/duckdb/tools/shell/linenoise/CMakeLists.txt

@@ -0,0 +1,7 @@
+include_directories(include)
+
+add_library(duckdb_linenoise OBJECT highlighting.cpp history.cpp linenoise.cpp
+                                    linenoise-c.cpp rendering.cpp terminal.cpp)
+set(SHELL_SOURCES
+    ${SHELL_SOURCES} $<TARGET_OBJECTS:duckdb_linenoise>
+    PARENT_SCOPE)

external/duckdb/tools/shell/linenoise/LICENSE

@@ -0,0 +1,25 @@
+Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com>
+Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

external/duckdb/tools/shell/linenoise/highlighting.cpp

@@ -0,0 +1,270 @@
+#include "linenoise.hpp"
+#include "linenoise.h"
+#include "highlighting.hpp"
+#include "duckdb/parser/parser.hpp"
+#include "duckdb/common/string.hpp"
+
+#if defined(_WIN32) || defined(__WIN32__) || defined(WIN32)
+// disable highlighting on windows (for now?)
+#define DISABLE_HIGHLIGHT
+#endif
+
+namespace duckdb {
+
+#ifdef DISABLE_HIGHLIGHT
+static int enableHighlighting = 0;
+#else
+static int enableHighlighting = 1;
+#endif
+struct Color {
+	const char *color_name;
+	const char *highlight;
+};
+static Color terminal_colors[] = {{"red", "\033[31m"},           {"green", "\033[32m"},
+                                  {"yellow", "\033[33m"},        {"blue", "\033[34m"},
+                                  {"magenta", "\033[35m"},       {"cyan", "\033[36m"},
+                                  {"white", "\033[37m"},         {"brightblack", "\033[90m"},
+                                  {"brightred", "\033[91m"},     {"brightgreen", "\033[92m"},
+                                  {"brightyellow", "\033[93m"},  {"brightblue", "\033[94m"},
+                                  {"brightmagenta", "\033[95m"}, {"brightcyan", "\033[96m"},
+                                  {"brightwhite", "\033[97m"},   {nullptr, nullptr}};
+static std::string bold = "\033[1m";
+static std::string underline = "\033[4m";
+static std::string keyword = "\033[32m";
+static std::string continuation_selected = "\033[32m";
+static std::string constant = "\033[33m";
+static std::string continuation = "\033[90m";
+static std::string comment = "\033[90m";
+static std::string error = "\033[31m";
+static std::string reset = "\033[00m";
+
+void Highlighting::Enable() {
+	enableHighlighting = 1;
+}
+
+void Highlighting::Disable() {
+	enableHighlighting = 0;
+}
+
+bool Highlighting::IsEnabled() {
+	return enableHighlighting;
+}
+
+const char *Highlighting::GetColorOption(const char *option) {
+	size_t index = 0;
+	while (terminal_colors[index].color_name) {
+		if (strcmp(terminal_colors[index].color_name, option) == 0) {
+			return terminal_colors[index].highlight;
+		}
+		index++;
+	}
+	return nullptr;
+}
+
+void Highlighting::SetHighlightingColor(HighlightingType type, const char *color) {
+	switch (type) {
+	case HighlightingType::KEYWORD:
+		keyword = color;
+		break;
+	case HighlightingType::CONSTANT:
+		constant = color;
+		break;
+	case HighlightingType::COMMENT:
+		comment = color;
+		break;
+	case HighlightingType::ERROR:
+		error = color;
+		break;
+	case HighlightingType::CONTINUATION:
+		continuation = color;
+		break;
+	case HighlightingType::CONTINUATION_SELECTED:
+		continuation_selected = color;
+		break;
+	}
+}
+
+static tokenType convertToken(duckdb::SimplifiedTokenType token_type) {
+	switch (token_type) {
+	case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_IDENTIFIER:
+		return tokenType::TOKEN_IDENTIFIER;
+	case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_NUMERIC_CONSTANT:
+		return tokenType::TOKEN_NUMERIC_CONSTANT;
+	case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_STRING_CONSTANT:
+		return tokenType::TOKEN_STRING_CONSTANT;
+	case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_OPERATOR:
+		return tokenType::TOKEN_OPERATOR;
+	case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_KEYWORD:
+		return tokenType::TOKEN_KEYWORD;
+	case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_COMMENT:
+		return tokenType::TOKEN_COMMENT;
+	default:
+		throw duckdb::InternalException("Unrecognized token type");
+	}
+}
+
+static vector<highlightToken> GetParseTokens(char *buf, size_t len) {
+	string sql(buf, len);
+	auto parseTokens = duckdb::Parser::Tokenize(sql);
+
+	vector<highlightToken> tokens;
+	for (auto &token : parseTokens) {
+		highlightToken new_token;
+		new_token.type = convertToken(token.type);
+		new_token.start = token.start;
+		tokens.push_back(new_token);
+	}
+
+	if (!tokens.empty() && tokens[0].start > 0) {
+		highlightToken new_token;
+		new_token.type = tokenType::TOKEN_IDENTIFIER;
+		new_token.start = 0;
+		tokens.insert(tokens.begin(), new_token);
+	}
+	if (tokens.empty() && sql.size() > 0) {
+		highlightToken new_token;
+		new_token.type = tokenType::TOKEN_IDENTIFIER;
+		new_token.start = 0;
+		tokens.push_back(new_token);
+	}
+	return tokens;
+}
+
+static vector<highlightToken> GetDotCommandTokens(char *buf, size_t len) {
+	vector<highlightToken> tokens;
+
+	// identifier token for the dot command itself
+	highlightToken dot_token;
+	dot_token.type = tokenType::TOKEN_KEYWORD;
+	dot_token.start = 0;
+	tokens.push_back(dot_token);
+
+	for (idx_t i = 0; i + 1 < len; i++) {
+		if (Linenoise::IsSpace(buf[i])) {
+			highlightToken argument_token;
+			argument_token.type = tokenType::TOKEN_STRING_CONSTANT;
+			argument_token.start = i + 1;
+			tokens.push_back(argument_token);
+		}
+	}
+	return tokens;
+}
+
+vector<highlightToken> Highlighting::Tokenize(char *buf, size_t len, bool is_dot_command, searchMatch *match) {
+	vector<highlightToken> tokens;
+	if (!is_dot_command) {
+		// SQL query - use parser to obtain tokens
+		tokens = GetParseTokens(buf, len);
+	} else {
+		// . command
+		tokens = GetDotCommandTokens(buf, len);
+	}
+	if (match) {
+		// we have a search match - insert it into the token list
+		// we want to insert a search token with start = match_start, end = match_end
+		// first figure out which token type we would have at match_end (if any)
+		for (size_t i = 0; i + 1 < tokens.size(); i++) {
+			if (tokens[i].start <= match->match_start && tokens[i + 1].start >= match->match_start) {
+				// this token begins after the search position, insert the token here
+				size_t token_position = i + 1;
+				auto end_type = tokens[i].type;
+				if (tokens[i].start == match->match_start) {
+					// exact start: only set the search match
+					tokens[i].search_match = true;
+				} else {
+					// non-exact start: add a new token
+					highlightToken search_token;
+					search_token.type = tokens[i].type;
+					search_token.start = match->match_start;
+					search_token.search_match = true;
+					tokens.insert(tokens.begin() + token_position, search_token);
+					token_position++;
+				}
+
+				// move forwards
+				while (token_position < tokens.size() && tokens[token_position].start < match->match_end) {
+					// this token is
+					// mark this token as a search token
+					end_type = tokens[token_position].type;
+					tokens[token_position].search_match = true;
+					token_position++;
+				}
+				if (token_position >= tokens.size() || tokens[token_position].start > match->match_end) {
+					// insert the token that marks the end of the search
+					highlightToken end_token;
+					end_token.type = end_type;
+					end_token.start = match->match_end;
+					tokens.insert(tokens.begin() + token_position, end_token);
+					token_position++;
+				}
+				break;
+			}
+		}
+	}
+	return tokens;
+}
+
+string Highlighting::HighlightText(char *buf, size_t len, size_t start_pos, size_t end_pos,
+                                   const vector<highlightToken> &tokens) {
+	duckdb::stringstream ss;
+	size_t prev_pos = 0;
+	for (size_t i = 0; i < tokens.size(); i++) {
+		size_t next = i + 1 < tokens.size() ? tokens[i + 1].start : len;
+		if (next < start_pos) {
+			// this token is not rendered at all
+			continue;
+		}
+
+		auto &token = tokens[i];
+		size_t start = token.start > start_pos ? token.start : start_pos;
+		size_t end = next > end_pos ? end_pos : next;
+		if (end <= start) {
+			continue;
+		}
+		if (prev_pos > start) {
+#ifdef DEBUG
+			throw InternalException("ERROR - Rendering at position %llu after rendering at position %llu\n", start,
+			                        prev_pos);
+#endif
+			Linenoise::Log("ERROR - Rendering at position %llu after rendering at position %llu\n", start, prev_pos);
+			continue;
+		}
+		prev_pos = start;
+		std::string text = std::string(buf + start, end - start);
+		if (token.search_match) {
+			ss << underline;
+		}
+		switch (token.type) {
+		case tokenType::TOKEN_KEYWORD:
+			ss << keyword << text << reset;
+			break;
+		case tokenType::TOKEN_NUMERIC_CONSTANT:
+		case tokenType::TOKEN_STRING_CONSTANT:
+			ss << constant << text << reset;
+			break;
+		case tokenType::TOKEN_CONTINUATION:
+			ss << continuation << text << reset;
+			break;
+		case tokenType::TOKEN_CONTINUATION_SELECTED:
+			ss << continuation_selected << text << reset;
+			break;
+		case tokenType::TOKEN_BRACKET:
+			ss << underline << text << reset;
+			break;
+		case tokenType::TOKEN_ERROR:
+			ss << error << text << reset;
+			break;
+		case tokenType::TOKEN_COMMENT:
+			ss << comment << text << reset;
+			break;
+		default:
+			ss << text;
+			if (token.search_match) {
+				ss << reset;
+			}
+		}
+	}
+	return ss.str();
+}
+
+} // namespace duckdb

external/duckdb/tools/shell/linenoise/history.cpp

@@ -0,0 +1,367 @@
+#include "history.hpp"
+#include "linenoise.hpp"
+#include "terminal.hpp"
+#include "duckdb_shell_wrapper.h"
+#include "sqlite3.h"
+#include "utf8proc_wrapper.hpp"
+#include <sys/stat.h>
+
+namespace duckdb {
+#define LINENOISE_DEFAULT_HISTORY_MAX_LEN 1000
+static idx_t history_max_len = LINENOISE_DEFAULT_HISTORY_MAX_LEN;
+static idx_t history_len = 0;
+static char **history = nullptr;
+static char *history_file = nullptr;
+
+/* Free the history, but does not reset it. Only used when we have to
+ * exit() to avoid memory leaks are reported by valgrind & co. */
+void History::Free() {
+	if (history) {
+		for (idx_t j = 0; j < history_len; j++) {
+			free(history[j]);
+		}
+		free(history);
+	}
+}
+
+idx_t History::GetLength() {
+	return history_len;
+}
+
+const char *History::GetEntry(idx_t index) {
+	if (!history || index >= history_len) {
+		// FIXME: print debug message
+		return "";
+	}
+	return history[index];
+}
+
+void History::Overwrite(idx_t index, const char *new_entry) {
+	if (!history || index >= history_len) {
+		// FIXME: print debug message
+		return;
+	}
+
+	free(history[index]);
+	history[index] = strdup(new_entry);
+}
+
+void History::RemoveLastEntry() {
+	history_len--;
+	free(history[history_len]);
+}
+
+int History::Add(const char *line) {
+	return History::Add(line, strlen(line));
+}
+
+int History::Add(const char *line, idx_t len) {
+	char *linecopy;
+
+	if (history_max_len == 0) {
+		return 0;
+	}
+
+	/* Initialization on first call. */
+	if (history == nullptr) {
+		history = (char **)malloc(sizeof(char *) * history_max_len);
+		if (history == nullptr) {
+			return 0;
+		}
+		memset(history, 0, (sizeof(char *) * history_max_len));
+	}
+
+	/* Don't add duplicated lines. */
+	if (history_len && !strcmp(history[history_len - 1], line)) {
+		return 0;
+	}
+
+	if (!Utf8Proc::IsValid(line, len)) {
+		// don't add invalid UTF8 to history
+		return 0;
+	}
+
+	/* Add an heap allocated copy of the line in the history.
+	 * If we reached the max length, remove the older line. */
+	if (!Terminal::IsMultiline()) {
+		// replace all newlines with spaces
+		linecopy = strdup(line);
+		if (!linecopy) {
+			return 0;
+		}
+		for (auto ptr = linecopy; *ptr; ptr++) {
+			if (*ptr == '\n' || *ptr == '\r') {
+				*ptr = ' ';
+			}
+		}
+	} else {
+		// replace all '\n' with '\r\n'
+		idx_t replaced_newline_count = 0;
+		idx_t len;
+		for (len = 0; line[len]; len++) {
+			if (line[len] == '\r' && line[len + 1] == '\n') {
+				// \r\n - skip past the \n
+				len++;
+			} else if (line[len] == '\n') {
+				replaced_newline_count++;
+			}
+		}
+		linecopy = (char *)malloc((len + replaced_newline_count + 1) * sizeof(char));
+		idx_t pos = 0;
+		for (len = 0; line[len]; len++) {
+			if (line[len] == '\r' && line[len + 1] == '\n') {
+				// \r\n - skip past the \n
+				linecopy[pos++] = '\r';
+				len++;
+			} else if (line[len] == '\n') {
+				linecopy[pos++] = '\r';
+			}
+			linecopy[pos++] = line[len];
+		}
+		linecopy[pos] = '\0';
+	}
+	if (history_len == history_max_len) {
+		free(history[0]);
+		memmove(history, history + 1, sizeof(char *) * (history_max_len - 1));
+		history_len--;
+	}
+	history[history_len] = linecopy;
+	history_len++;
+	if (history_file && strlen(line) > 0) {
+		// if there is a history file that we loaded from
+		// append to the history
+		// this way we can recover history in case of a crash
+		FILE *fp;
+
+		fp = fopen(history_file, "a");
+		if (fp == nullptr) {
+			return 1;
+		}
+		fprintf(fp, "%s\n", line);
+		fclose(fp);
+	}
+	return 1;
+}
+
+int History::SetMaxLength(idx_t len) {
+	char **new_entry;
+
+	if (len < 1) {
+		return 0;
+	}
+	if (history) {
+		idx_t tocopy = history_len;
+
+		new_entry = (char **)malloc(sizeof(char *) * len);
+		if (new_entry == nullptr) {
+			return 0;
+		}
+
+		/* If we can't copy everything, free the elements we'll not use. */
+		if (len < tocopy) {
+			for (idx_t j = 0; j < tocopy - len; j++) {
+				free(history[j]);
+			}
+			tocopy = len;
+		}
+		memset(new_entry, 0, sizeof(char *) * len);
+		memcpy(new_entry, history + (history_len - tocopy), sizeof(char *) * tocopy);
+		free(history);
+		history = new_entry;
+	}
+	history_max_len = len;
+	if (history_len > history_max_len) {
+		history_len = history_max_len;
+	}
+	return 1;
+}
+
+int History::Save(const char *filename) {
+	mode_t old_umask = umask(S_IXUSR | S_IRWXG | S_IRWXO);
+	FILE *fp;
+
+	fp = fopen(filename, "w");
+	umask(old_umask);
+	if (fp == nullptr) {
+		return -1;
+	}
+	chmod(filename, S_IRUSR | S_IWUSR);
+	for (idx_t j = 0; j < history_len; j++) {
+		fprintf(fp, "%s\n", history[j]);
+	}
+	fclose(fp);
+	return 0;
+}
+
+struct LineReader {
+	static constexpr idx_t LINE_BUFFER_SIZE = LINENOISE_MAX_LINE * 2ULL;
+
+public:
+	LineReader() : fp(nullptr), filename(nullptr), end_of_file(false), position(0), capacity(0), total_read(0) {
+		line_buffer[LINENOISE_MAX_LINE] = '\0';
+		data_buffer[LINE_BUFFER_SIZE] = '\0';
+	}
+
+	bool Init(const char *filename_p) {
+		filename = filename_p;
+		fp = fopen(filename, "r");
+		return fp;
+	}
+
+	void Close() {
+		if (fp) {
+			fclose(fp);
+			fp = nullptr;
+		}
+	}
+
+	const char *GetLine() {
+		return line_buffer;
+	}
+
+	idx_t GetNextNewline() {
+		for (idx_t i = position; i < capacity; i++) {
+			if (data_buffer[i] == '\r' || data_buffer[i] == '\n') {
+				return i;
+			}
+		}
+		return capacity;
+	}
+
+	void SkipNewline() {
+		if (position >= capacity) {
+			// we are already at the end - fill the buffer
+			FillBuffer();
+		}
+		if (position < capacity && data_buffer[position] == '\n') {
+			position++;
+		}
+	}
+
+	bool NextLine() {
+		idx_t line_size = 0;
+		while (true) {
+			// find the next newline in the current buffer (if any)
+			idx_t i = GetNextNewline();
+			// copy over the data and move to the next byte
+			idx_t read_count = i - position;
+			if (line_size + read_count > LINENOISE_MAX_LINE) {
+				// exceeded max line size
+				// move on to next line and don't add to history
+				// skip to next newline
+				bool found_next_newline = false;
+				while (!found_next_newline && capacity > 0) {
+					i = GetNextNewline();
+					if (i < capacity) {
+						found_next_newline = true;
+					}
+					if (!found_next_newline) {
+						// read more data
+						FillBuffer();
+					}
+				}
+				if (!found_next_newline) {
+					// no newline found - skip
+					return false;
+				}
+				// newline found - adjust position and read next line
+				position = i + 1;
+				if (data_buffer[i] == '\r') {
+					// \r\n - skip the next byte as well
+					SkipNewline();
+				}
+				continue;
+			}
+			memcpy(line_buffer + line_size, data_buffer + position, read_count);
+			line_size += read_count;
+
+			if (i < capacity) {
+				// we're still within the buffer - this means we found a newline in the buffer
+				line_buffer[line_size] = '\0';
+				position = i + 1;
+				if (data_buffer[i] == '\r') {
+					// \r\n - skip the next byte as well
+					SkipNewline();
+				}
+				if (line_size == 0 || !Utf8Proc::IsValid(line_buffer, line_size)) {
+					// line is empty OR not valid UTF8
+					// move on to next line and don't add to history
+					line_size = 0;
+					continue;
+				}
+				return true;
+			}
+			// we need to read more data - fill up the buffer
+			FillBuffer();
+			if (capacity == 0) {
+				// no more data available - return true if there is anything we copied over (i.e. part of the next line)
+				return line_size > 0;
+			}
+		}
+	}
+
+	void FillBuffer() {
+		if (end_of_file || !fp) {
+			return;
+		}
+		size_t read_data = fread(data_buffer, 1, LINE_BUFFER_SIZE, fp);
+		position = 0;
+		capacity = read_data;
+		total_read += read_data;
+		data_buffer[read_data] = '\0';
+
+		if (read_data == 0) {
+			end_of_file = true;
+		}
+		if (total_read >= LINENOISE_MAX_HISTORY) {
+			fprintf(stderr, "History file \"%s\" exceeds maximum history file size of %d MB - skipping full load\n",
+			        filename, LINENOISE_MAX_HISTORY / 1024 / 1024);
+			capacity = 0;
+			end_of_file = true;
+		}
+	}
+
+private:
+	FILE *fp;
+	const char *filename;
+	char line_buffer[LINENOISE_MAX_LINE + 1];
+	char data_buffer[LINE_BUFFER_SIZE + 1];
+	bool end_of_file;
+	idx_t position;
+	idx_t capacity;
+	idx_t total_read;
+};
+
+int History::Load(const char *filename) {
+	LineReader reader;
+	if (!reader.Init(filename)) {
+		return -1;
+	}
+
+	std::string result;
+	while (reader.NextLine()) {
+		auto buf = reader.GetLine();
+		if (result.empty() && buf[0] == '.') {
+			// if the first character is a dot this is a dot command
+			// add the full line to the history
+			History::Add(buf);
+			continue;
+		}
+		// else we are parsing a SQL statement
+		result += buf;
+		if (sqlite3_complete(result.c_str())) {
+			// this line contains a full SQL statement - add it to the history
+			History::Add(result.c_str(), result.size());
+			result = std::string();
+			continue;
+		}
+		// the result does not contain a full SQL statement - add a newline deliminator and move on to the next line
+		result += "\r\n";
+	}
+	reader.Close();
+
+	history_file = strdup(filename);
+	return 0;
+}
+
+} // namespace duckdb

external/duckdb/tools/shell/linenoise/include/highlighting.hpp

@@ -0,0 +1,50 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// highlighting.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+
+namespace duckdb {
+struct searchMatch;
+
+enum class tokenType : uint8_t {
+	TOKEN_IDENTIFIER,
+	TOKEN_NUMERIC_CONSTANT,
+	TOKEN_STRING_CONSTANT,
+	TOKEN_OPERATOR,
+	TOKEN_KEYWORD,
+	TOKEN_COMMENT,
+	TOKEN_CONTINUATION,
+	TOKEN_CONTINUATION_SELECTED,
+	TOKEN_BRACKET,
+	TOKEN_ERROR
+};
+
+enum class HighlightingType { KEYWORD, CONSTANT, COMMENT, ERROR, CONTINUATION, CONTINUATION_SELECTED };
+
+struct highlightToken {
+	tokenType type;
+	size_t start = 0;
+	bool search_match = false;
+};
+
+class Highlighting {
+public:
+	static void Enable();
+	static void Disable();
+	static bool IsEnabled();
+	static const char *GetColorOption(const char *option);
+	static void SetHighlightingColor(HighlightingType type, const char *color);
+
+	static vector<highlightToken> Tokenize(char *buf, size_t len, bool is_dot_command, searchMatch *match);
+	static string HighlightText(char *buf, size_t len, size_t start_pos, size_t end_pos,
+	                            const vector<highlightToken> &tokens);
+};
+
+} // namespace duckdb

external/duckdb/tools/shell/linenoise/include/history.hpp

@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// history.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+
+namespace duckdb {
+
+class History {
+public:
+	static void Free();
+	static idx_t GetLength();
+	static const char *GetEntry(idx_t index);
+	static void Overwrite(idx_t index, const char *new_entry);
+	static void RemoveLastEntry();
+	static int Add(const char *line);
+	static int Add(const char *line, idx_t len);
+	static int SetMaxLength(idx_t len);
+	static int Save(const char *filename);
+	static int Load(const char *filename);
+};
+
+} // namespace duckdb

external/duckdb/tools/shell/linenoise/include/linenoise.h

@@ -0,0 +1,76 @@
+/* linenoise.h -- VERSION 1.0
+ *
+ * Guerrilla line editing library against the idea that a line editing lib
+ * needs to be 20,000 lines of C code.
+ *
+ * See linenoise.c for more information.
+ *
+ * ------------------------------------------------------------------------
+ *
+ * Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *  *  Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *
+ *  *  Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __LINENOISE_H
+#define __LINENOISE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct linenoiseCompletions {
+	size_t len;
+	char **cvec;
+} linenoiseCompletions;
+
+typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *);
+typedef char *(linenoiseHintsCallback)(const char *, int *color, int *bold);
+typedef void(linenoiseFreeHintsCallback)(void *);
+void linenoiseSetCompletionCallback(linenoiseCompletionCallback *);
+void linenoiseSetHintsCallback(linenoiseHintsCallback *);
+void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *);
+void linenoiseAddCompletion(linenoiseCompletions *, const char *);
+
+char *linenoise(const char *prompt);
+void linenoiseFree(void *ptr);
+int linenoiseParseOption(const char **azArg, int nArg, const char **out_error);
+int linenoiseHistoryAdd(const char *line);
+int linenoiseHistorySetMaxLen(int len);
+int linenoiseHistorySave(const char *filename);
+int linenoiseHistoryLoad(const char *filename);
+void linenoiseClearScreen(void);
+void linenoiseSetMultiLine(int ml);
+size_t linenoiseComputeRenderWidth(const char *buf, size_t len);
+int linenoiseGetRenderPosition(const char *buf, size_t len, int max_width, int *n);
+void linenoiseSetPrompt(const char *continuation, const char *continuationSelected);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __LINENOISE_H */

external/duckdb/tools/shell/linenoise/include/linenoise.hpp

@@ -0,0 +1,190 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// linenoise.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/exception.hpp"
+#include "terminal.hpp"
+#include "linenoise.h"
+
+#define LINENOISE_MAX_LINE    204800
+#define LINENOISE_MAX_HISTORY 104857600
+#define LINENOISE_EDITOR
+
+namespace duckdb {
+struct highlightToken;
+struct AppendBuffer;
+
+enum class HistoryScrollDirection : uint8_t {
+	LINENOISE_HISTORY_NEXT,
+	LINENOISE_HISTORY_PREV,
+	LINENOISE_HISTORY_START,
+	LINENOISE_HISTORY_END
+};
+enum class Capitalization : uint8_t { CAPITALIZE, LOWERCASE, UPPERCASE };
+
+struct searchMatch {
+	size_t history_index;
+	size_t match_start;
+	size_t match_end;
+};
+
+struct Completion {
+	string completion;
+	idx_t cursor_pos;
+};
+
+struct TabCompletion {
+	vector<Completion> completions;
+};
+
+class Linenoise {
+public:
+	Linenoise(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt);
+
+public:
+	int Edit();
+
+	static void SetCompletionCallback(linenoiseCompletionCallback *fn);
+	static void SetHintsCallback(linenoiseHintsCallback *fn);
+	static void SetFreeHintsCallback(linenoiseFreeHintsCallback *fn);
+
+	static linenoiseHintsCallback *HintsCallback();
+	static linenoiseFreeHintsCallback *FreeHintsCallback();
+
+	static void SetPrompt(const char *continuation, const char *continuationSelected);
+	static size_t ComputeRenderWidth(const char *buf, size_t len);
+	static int GetRenderPosition(const char *buf, size_t len, int max_width, int *n);
+
+	static int ParseOption(const char **azArg, int nArg, const char **out_error);
+
+	int GetPromptWidth() const;
+
+	void RefreshLine();
+	int CompleteLine(EscapeSequence &current_sequence);
+	void InsertCharacter(char c);
+	int EditInsert(char c);
+	int EditInsertMulti(const char *c);
+	void EditMoveLeft();
+	void EditMoveRight();
+	void EditMoveWordLeft();
+	void EditMoveWordRight();
+	bool EditMoveRowUp();
+	bool EditMoveRowDown();
+	void EditMoveHome();
+	void EditMoveEnd();
+	void EditMoveStartOfLine();
+	void EditMoveEndOfLine();
+	void EditHistoryNext(HistoryScrollDirection dir);
+	void EditHistorySetIndex(idx_t index);
+	void EditDelete();
+	void EditBackspace();
+	void EditDeletePrevWord();
+	void EditDeleteNextWord();
+	void EditDeleteAll();
+	void EditCapitalizeNextWord(Capitalization capitalization);
+	void EditRemoveSpaces();
+	void EditSwapCharacter();
+	void EditSwapWord();
+
+	void StartSearch();
+	void CancelSearch();
+	char AcceptSearch(char nextCommand);
+	void PerformSearch();
+	void SearchPrev();
+	void SearchNext();
+
+#ifdef LINENOISE_EDITOR
+	bool EditBufferWithEditor(const char *editor);
+	bool EditFileWithEditor(const string &file_name, const char *editor);
+#endif
+
+	char Search(char c);
+
+	void RefreshMultiLine();
+	void RefreshSingleLine() const;
+	void RefreshSearch();
+	void RefreshShowHints(AppendBuffer &append_buffer, int plen) const;
+
+	size_t PrevChar() const;
+	size_t NextChar() const;
+
+	void NextPosition(const char *buf, size_t len, size_t &cpos, int &rows, int &cols, int plen) const;
+	void PositionToColAndRow(size_t target_pos, int &out_row, int &out_col, int &rows, int &cols) const;
+	size_t ColAndRowToPosition(int target_row, int target_col) const;
+
+	string AddContinuationMarkers(const char *buf, size_t len, int plen, int cursor_row,
+	                              vector<highlightToken> &tokens) const;
+	void AddErrorHighlighting(idx_t render_start, idx_t render_end, vector<highlightToken> &tokens) const;
+
+	bool AddCompletionMarker(const char *buf, idx_t len, string &result_buffer, vector<highlightToken> &tokens) const;
+
+	static bool IsNewline(char c);
+	static bool IsWordBoundary(char c);
+	static bool AllWhitespace(const char *z);
+	static bool IsSpace(char c);
+
+	TabCompletion TabComplete() const;
+
+	static void EnableCompletionRendering();
+	static void DisableCompletionRendering();
+	static void EnableErrorRendering();
+	static void DisableErrorRendering();
+
+public:
+	static void LogTokens(const vector<highlightToken> &tokens);
+#ifdef LINENOISE_LOGGING
+	// Logging
+	template <typename... Args>
+	static void Log(const string &msg, Args... params) {
+		std::vector<ExceptionFormatValue> values;
+		LogMessageRecursive(msg, values, params...);
+	}
+
+	static void LogMessageRecursive(const string &msg, std::vector<ExceptionFormatValue> &values);
+
+	template <class T, typename... Args>
+	static void LogMessageRecursive(const string &msg, std::vector<ExceptionFormatValue> &values, T param,
+	                                Args... params) {
+		values.push_back(ExceptionFormatValue::CreateFormatValue<T>(param));
+		LogMessageRecursive(msg, values, params...);
+	}
+#else
+	template <typename... Args>
+	static void Log(const string &msg, Args... params) {
+		// nop
+	}
+#endif
+
+public:
+	int ifd;                                 /* Terminal stdin file descriptor. */
+	int ofd;                                 /* Terminal stdout file descriptor. */
+	char *buf;                               /* Edited line buffer. */
+	size_t buflen;                           /* Edited line buffer size. */
+	const char *prompt;                      /* Prompt to display. */
+	size_t plen;                             /* Prompt length. */
+	size_t pos;                              /* Current cursor position. */
+	size_t old_cursor_rows;                  /* Previous refresh cursor position. */
+	size_t len;                              /* Current edited line length. */
+	size_t y_scroll;                         /* The y scroll position (multiline mode) */
+	TerminalSize ws;                         /* Terminal size */
+	size_t maxrows;                          /* Maximum num of rows used so far (multiline mode) */
+	idx_t history_index;                     /* The history index we are currently editing. */
+	bool clear_screen;                       /* Whether we are clearing the screen */
+	bool continuation_markers;               /* Whether or not to render continuation markers */
+	bool search;                             /* Whether or not we are searching our history */
+	bool render;                             /* Whether or not to re-render */
+	bool has_more_data;                      /* Whether or not there is more data available in the buffer (copy+paste)*/
+	bool insert;                             /* Whether or not the last action was inserting a new character */
+	std::string search_buf;                  //! The search buffer
+	std::vector<searchMatch> search_matches; //! The set of search matches in our history
+	size_t search_index;                     //! The current match index
+};
+
+} // namespace duckdb

external/duckdb/tools/shell/linenoise/include/terminal.hpp

@@ -0,0 +1,122 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// terminal.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+
+namespace duckdb {
+
+enum KEY_ACTION {
+	KEY_NULL = 0,   /* NULL */
+	CTRL_A = 1,     /* Ctrl+a */
+	CTRL_B = 2,     /* Ctrl-b */
+	CTRL_C = 3,     /* Ctrl-c */
+	CTRL_D = 4,     /* Ctrl-d */
+	CTRL_E = 5,     /* Ctrl-e */
+	CTRL_F = 6,     /* Ctrl-f */
+	CTRL_G = 7,     /* Ctrl-g */
+	CTRL_H = 8,     /* Ctrl-h */
+	TAB = 9,        /* Tab */
+	CTRL_J = 10,    /* Ctrl+j*/
+	CTRL_K = 11,    /* Ctrl+k */
+	CTRL_L = 12,    /* Ctrl+l */
+	ENTER = 13,     /* Enter */
+	CTRL_N = 14,    /* Ctrl-n */
+	CTRL_O = 15,    /* Ctrl-O */
+	CTRL_P = 16,    /* Ctrl-p */
+	CTRL_R = 18,    /* Ctrl-r */
+	CTRL_S = 19,    /* Ctrl-s */
+	CTRL_T = 20,    /* Ctrl-t */
+	CTRL_U = 21,    /* Ctrl+u */
+	CTRL_W = 23,    /* Ctrl+w */
+	CTRL_X = 24,    /* Ctrl+x */
+	CTRL_Y = 25,    /* Ctrl+y */
+	CTRL_Z = 26,    /* Ctrl+z */
+	ESC = 27,       /* Escape */
+	BACKSPACE = 127 /* Backspace */
+};
+
+enum class EscapeSequence {
+	INVALID = 0,
+	UNKNOWN = 1,
+	CTRL_MOVE_BACKWARDS,
+	CTRL_MOVE_FORWARDS,
+	HOME,
+	END,
+	UP,
+	DOWN,
+	RIGHT,
+	LEFT,
+	DELETE,
+	SHIFT_TAB,
+	ESCAPE,
+	ALT_A,
+	ALT_B,
+	ALT_C,
+	ALT_D,
+	ALT_E,
+	ALT_F,
+	ALT_G,
+	ALT_H,
+	ALT_I,
+	ALT_J,
+	ALT_K,
+	ALT_L,
+	ALT_M,
+	ALT_N,
+	ALT_O,
+	ALT_P,
+	ALT_Q,
+	ALT_R,
+	ALT_S,
+	ALT_T,
+	ALT_U,
+	ALT_V,
+	ALT_W,
+	ALT_X,
+	ALT_Y,
+	ALT_Z,
+	ALT_BACKSPACE,
+	ALT_LEFT_ARROW,
+	ALT_RIGHT_ARROW,
+	ALT_BACKSLASH,
+};
+
+struct TerminalSize {
+	int ws_col = 0;
+	int ws_row = 0;
+};
+
+class Terminal {
+public:
+	static int IsUnsupportedTerm();
+	static int EnableRawMode();
+	static void DisableRawMode();
+	static bool IsMultiline();
+	static void SetMultiLine(int ml);
+
+	static void ClearScreen();
+	static void Beep();
+
+	static bool IsAtty();
+	static int HasMoreData(int fd);
+	static TerminalSize GetTerminalSize();
+
+	static char *EditNoTTY();
+	static int EditRaw(char *buf, size_t buflen, const char *prompt);
+
+	static EscapeSequence ReadEscapeSequence(int ifd);
+
+private:
+	static TerminalSize TryMeasureTerminalSize();
+	static TerminalSize GetCursorPosition();
+	static idx_t ReadEscapeSequence(int ifd, char sequence[]);
+};
+
+} // namespace duckdb

external/duckdb/tools/shell/linenoise/linenoise-c.cpp

@@ -0,0 +1,154 @@
+#include "linenoise.hpp"
+#include "linenoise.h"
+#include "history.hpp"
+#include "terminal.hpp"
+
+using duckdb::History;
+using duckdb::idx_t;
+using duckdb::Linenoise;
+using duckdb::Terminal;
+
+/* The high level function that is the main API of the linenoise library.
+ * This function checks if the terminal has basic capabilities, just checking
+ * for a blacklist of stupid terminals, and later either calls the line
+ * editing function or uses dummy fgets() so that you will be able to type
+ * something even in the most desperate of the conditions. */
+char *linenoise(const char *prompt) {
+	char buf[LINENOISE_MAX_LINE];
+	int count;
+
+	if (!Terminal::IsAtty()) {
+		/* Not a tty: read from file / pipe. In this mode we don't want any
+		 * limit to the line size, so we call a function to handle that. */
+		return Terminal::EditNoTTY();
+	} else if (Terminal::IsUnsupportedTerm()) {
+		size_t len;
+
+		printf("%s", prompt);
+		fflush(stdout);
+		if (fgets(buf, LINENOISE_MAX_LINE, stdin) == NULL) {
+			return NULL;
+		}
+		len = strlen(buf);
+		while (len && (buf[len - 1] == '\n' || buf[len - 1] == '\r')) {
+			len--;
+			buf[len] = '\0';
+		}
+		return strdup(buf);
+	} else {
+		count = Terminal::EditRaw(buf, LINENOISE_MAX_LINE, prompt);
+		if (count == -1) {
+			return NULL;
+		}
+		return strdup(buf);
+	}
+}
+
+/* This is just a wrapper the user may want to call in order to make sure
+ * the linenoise returned buffer is freed with the same allocator it was
+ * created with. Useful when the main program is using an alternative
+ * allocator. */
+void linenoiseFree(void *ptr) {
+	free(ptr);
+}
+
+/* ================================ History ================================= */
+
+/* This is the API call to add a new entry in the linenoise history.
+ * It uses a fixed array of char pointers that are shifted (memmoved)
+ * when the history max length is reached in order to remove the older
+ * entry and make room for the new one, so it is not exactly suitable for huge
+ * histories, but will work well for a few hundred of entries.
+ *
+ * Using a circular buffer is smarter, but a bit more complex to handle. */
+int linenoiseHistoryAdd(const char *line) {
+	return History::Add(line);
+}
+
+/* Set the maximum length for the history. This function can be called even
+ * if there is already some history, the function will make sure to retain
+ * just the latest 'len' elements if the new history length value is smaller
+ * than the amount of items already inside the history. */
+int linenoiseHistorySetMaxLen(int len) {
+	if (len < 0) {
+		return 0;
+	}
+	return History::SetMaxLength(idx_t(len));
+}
+
+/* Save the history in the specified file. On success 0 is returned
+ * otherwise -1 is returned. */
+int linenoiseHistorySave(const char *filename) {
+	return History::Save(filename);
+}
+
+/* Load the history from the specified file. If the file does not exist
+ * zero is returned and no operation is performed.
+ *
+ * If the file exists and the operation succeeded 0 is returned, otherwise
+ * on error -1 is returned. */
+int linenoiseHistoryLoad(const char *filename) {
+	return History::Load(filename);
+}
+
+/* Register a callback function to be called for tab-completion. */
+void linenoiseSetCompletionCallback(linenoiseCompletionCallback *fn) {
+	Linenoise::SetCompletionCallback(fn);
+}
+
+/* Register a hits function to be called to show hits to the user at the
+ * right of the prompt. */
+void linenoiseSetHintsCallback(linenoiseHintsCallback *fn) {
+	Linenoise::SetHintsCallback(fn);
+}
+
+/* Register a function to free the hints returned by the hints callback
+ * registered with linenoiseSetHintsCallback(). */
+void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *fn) {
+	Linenoise::SetFreeHintsCallback(fn);
+}
+
+void linenoiseSetMultiLine(int ml) {
+	Terminal::SetMultiLine(ml);
+}
+
+void linenoiseSetPrompt(const char *continuation, const char *continuationSelected) {
+	Linenoise::SetPrompt(continuation, continuationSelected);
+}
+
+/* This function is used by the callback function registered by the user
+ * in order to add completion options given the input string when the
+ * user typed <tab>. See the example.c source code for a very easy to
+ * understand example. */
+void linenoiseAddCompletion(linenoiseCompletions *lc, const char *str) {
+	size_t len = strlen(str);
+	char *copy, **cvec;
+
+	copy = (char *)malloc(len + 1);
+	if (copy == NULL)
+		return;
+	memcpy(copy, str, len + 1);
+	cvec = (char **)realloc(lc->cvec, sizeof(char *) * (lc->len + 1));
+	if (cvec == NULL) {
+		free(copy);
+		return;
+	}
+	lc->cvec = cvec;
+	lc->cvec[lc->len++] = copy;
+}
+
+size_t linenoiseComputeRenderWidth(const char *buf, size_t len) {
+	return Linenoise::ComputeRenderWidth(buf, len);
+}
+
+int linenoiseGetRenderPosition(const char *buf, size_t len, int max_width, int *n) {
+	return Linenoise::GetRenderPosition(buf, len, max_width, n);
+}
+
+void linenoiseClearScreen(void) {
+	Terminal::ClearScreen();
+}
+
+int linenoiseParseOption(const char **azArg, int nArg, const char **out_error) {
+	return Linenoise::ParseOption(azArg, nArg, out_error);
+}

external/duckdb/tools/shell/linenoise/rendering.cpp

@@ -0,0 +1,975 @@
+#include "linenoise.hpp"
+#include "highlighting.hpp"
+#include "history.hpp"
+#include "utf8proc_wrapper.hpp"
+#include <unistd.h>
+
+namespace duckdb {
+static const char *continuationPrompt = "> ";
+static const char *continuationSelectedPrompt = "> ";
+static bool enableCompletionRendering = false;
+static bool enableErrorRendering = true;
+
+void Linenoise::EnableCompletionRendering() {
+	enableCompletionRendering = true;
+}
+
+void Linenoise::DisableCompletionRendering() {
+	enableCompletionRendering = false;
+}
+
+void Linenoise::EnableErrorRendering() {
+	enableErrorRendering = true;
+}
+
+void Linenoise::DisableErrorRendering() {
+	enableErrorRendering = false;
+}
+
+/* =========================== Line editing ================================= */
+
+/* We define a very simple "append buffer" structure, that is an heap
+ * allocated string where we can append to. This is useful in order to
+ * write all the escape sequences in a buffer and flush them to the standard
+ * output in a single call, to avoid flickering effects. */
+struct AppendBuffer {
+	void Append(const char *s, idx_t len) {
+		buffer.append(s, len);
+	}
+	void Append(const char *s) {
+		buffer.append(s);
+	}
+
+	void Write(int fd) {
+		if (write(fd, buffer.c_str(), buffer.size()) == -1) {
+			/* Can't recover from write error. */
+			Linenoise::Log("%s", "Failed to write buffer\n");
+		}
+	}
+
+private:
+	std::string buffer;
+};
+
+void Linenoise::SetPrompt(const char *continuation, const char *continuationSelected) {
+	continuationPrompt = continuation;
+	continuationSelectedPrompt = continuationSelected;
+}
+
+/* Helper of refreshSingleLine() and refreshMultiLine() to show hints
+ * to the right of the prompt. */
+void Linenoise::RefreshShowHints(AppendBuffer &append_buffer, int plen) const {
+	char seq[64];
+	auto hints_callback = Linenoise::HintsCallback();
+	if (hints_callback && plen + len < size_t(ws.ws_col)) {
+		int color = -1, bold = 0;
+		char *hint = hints_callback(buf, &color, &bold);
+		if (hint) {
+			int hintlen = strlen(hint);
+			int hintmaxlen = ws.ws_col - (plen + len);
+			if (hintlen > hintmaxlen) {
+				hintlen = hintmaxlen;
+			}
+			if (bold == 1 && color == -1)
+				color = 37;
+			if (color != -1 || bold != 0) {
+				snprintf(seq, 64, "\033[%d;%d;49m", bold, color);
+			} else {
+				seq[0] = '\0';
+			}
+			append_buffer.Append(seq, strlen(seq));
+			append_buffer.Append(hint, hintlen);
+			if (color != -1 || bold != 0) {
+				append_buffer.Append("\033[0m");
+			}
+			/* Call the function to free the hint returned. */
+			auto free_hints_callback = Linenoise::FreeHintsCallback();
+			if (free_hints_callback) {
+				free_hints_callback(hint);
+			}
+		}
+	}
+}
+
+static void renderText(size_t &render_pos, char *&buf, size_t &len, size_t pos, size_t cols, size_t plen,
+                       std::string &highlight_buffer, bool highlight, searchMatch *match = nullptr) {
+	if (duckdb::Utf8Proc::IsValid(buf, len)) {
+		// utf8 in prompt, handle rendering
+		size_t remaining_render_width = cols - plen - 1;
+		size_t start_pos = 0;
+		size_t cpos = 0;
+		size_t prev_pos = 0;
+		size_t total_render_width = 0;
+		while (cpos < len) {
+			size_t char_render_width = duckdb::Utf8Proc::RenderWidth(buf, len, cpos);
+			prev_pos = cpos;
+			cpos = duckdb::Utf8Proc::NextGraphemeCluster(buf, len, cpos);
+			total_render_width += cpos - prev_pos;
+			if (total_render_width >= remaining_render_width) {
+				// character does not fit anymore! we need to figure something out
+				if (prev_pos >= pos) {
+					// we passed the cursor: break
+					cpos = prev_pos;
+					break;
+				} else {
+					// we did not pass the cursor yet! remove characters from the start until it fits again
+					while (total_render_width >= remaining_render_width) {
+						size_t start_char_width = duckdb::Utf8Proc::RenderWidth(buf, len, start_pos);
+						size_t new_start = duckdb::Utf8Proc::NextGraphemeCluster(buf, len, start_pos);
+						total_render_width -= new_start - start_pos;
+						start_pos = new_start;
+						render_pos -= start_char_width;
+					}
+				}
+			}
+			if (prev_pos < pos) {
+				render_pos += char_render_width;
+			}
+		}
+		if (highlight) {
+			bool is_dot_command = buf[0] == '.';
+
+			auto tokens = Highlighting::Tokenize(buf, len, is_dot_command, match);
+			highlight_buffer = Highlighting::HighlightText(buf, len, start_pos, cpos, tokens);
+			buf = (char *)highlight_buffer.c_str();
+			len = highlight_buffer.size();
+		} else {
+			buf = buf + start_pos;
+			len = cpos - start_pos;
+		}
+	} else {
+		// invalid UTF8: fallback
+		while ((plen + pos) >= cols) {
+			buf++;
+			len--;
+			pos--;
+		}
+		while (plen + len > cols) {
+			len--;
+		}
+		render_pos = pos;
+	}
+}
+
+/* Single line low level line refresh.
+ *
+ * Rewrite the currently edited line accordingly to the buffer content,
+ * cursor position, and number of columns of the terminal. */
+void Linenoise::RefreshSingleLine() const {
+	char seq[64];
+	size_t plen = GetPromptWidth();
+	int fd = ofd;
+	char *render_buf = buf;
+	size_t render_len = len;
+	size_t render_pos = 0;
+	std::string highlight_buffer;
+
+	renderText(render_pos, render_buf, render_len, pos, ws.ws_col, plen, highlight_buffer, Highlighting::IsEnabled());
+
+	AppendBuffer append_buffer;
+	/* Cursor to left edge */
+	append_buffer.Append("\r");
+	/* Write the prompt and the current buffer content */
+	append_buffer.Append(prompt);
+	append_buffer.Append(render_buf, render_len);
+	/* Show hits if any. */
+	RefreshShowHints(append_buffer, plen);
+	/* Erase to right */
+	append_buffer.Append("\x1b[0K");
+	/* Move cursor to original position. */
+	snprintf(seq, 64, "\r\x1b[%dC", (int)(render_pos + plen));
+	append_buffer.Append(seq);
+	append_buffer.Write(fd);
+}
+
+void Linenoise::RefreshSearch() {
+	std::string search_prompt;
+	static const size_t SEARCH_PROMPT_RENDER_SIZE = 28;
+	std::string no_matches_text = "(no matches)";
+	bool no_matches = search_index >= search_matches.size();
+	if (search_buf.empty()) {
+		search_prompt = "search" + std::string(SEARCH_PROMPT_RENDER_SIZE - 8, ' ') + "> ";
+		no_matches_text = "(type to search)";
+	} else {
+		std::string search_text;
+		std::string matches_text;
+		search_text += search_buf;
+		if (!no_matches) {
+			matches_text += std::to_string(search_index + 1);
+			matches_text += "/" + std::to_string(search_matches.size());
+		}
+		size_t search_text_length = ComputeRenderWidth(search_text.c_str(), search_text.size());
+		size_t matches_text_length = ComputeRenderWidth(matches_text.c_str(), matches_text.size());
+		size_t total_text_length = search_text_length + matches_text_length;
+		if (total_text_length < SEARCH_PROMPT_RENDER_SIZE - 2) {
+			// search text is short: we can render the entire search text
+			search_prompt = search_text;
+			search_prompt += std::string(SEARCH_PROMPT_RENDER_SIZE - 2 - total_text_length, ' ');
+			search_prompt += matches_text;
+			search_prompt += "> ";
+		} else {
+			// search text length is too long to fit: truncate
+			bool render_matches = matches_text_length < SEARCH_PROMPT_RENDER_SIZE - 8;
+			char *search_buf = (char *)search_text.c_str();
+			size_t search_len = search_text.size();
+			size_t search_render_pos = 0;
+			size_t max_render_size = SEARCH_PROMPT_RENDER_SIZE - 3;
+			if (render_matches) {
+				max_render_size -= matches_text_length;
+			}
+			std::string highlight_buffer;
+			renderText(search_render_pos, search_buf, search_len, search_len, max_render_size, 0, highlight_buffer,
+			           false);
+			search_prompt = std::string(search_buf, search_len);
+			for (size_t i = search_render_pos; i < max_render_size; i++) {
+				search_prompt += " ";
+			}
+			if (render_matches) {
+				search_prompt += matches_text;
+			}
+			search_prompt += "> ";
+		}
+	}
+	auto oldHighlighting = Highlighting::IsEnabled();
+	Linenoise clone = *this;
+	prompt = search_prompt.c_str();
+	plen = search_prompt.size();
+	if (no_matches || search_buf.empty()) {
+		// if there are no matches render the no_matches_text
+		buf = (char *)no_matches_text.c_str();
+		len = no_matches_text.size();
+		pos = 0;
+		// don't highlight the "no_matches" text
+		Highlighting::Disable();
+	} else {
+		// if there are matches render the current history item
+		auto search_match = search_matches[search_index];
+		auto history_index = search_match.history_index;
+		auto cursor_position = search_match.match_end;
+		buf = (char *)History::GetEntry(history_index);
+		len = strlen(buf);
+		pos = cursor_position;
+	}
+	RefreshLine();
+
+	if (oldHighlighting) {
+		Highlighting::Enable();
+	}
+	buf = clone.buf;
+	len = clone.len;
+	pos = clone.pos;
+	prompt = clone.prompt;
+	plen = clone.plen;
+}
+
+string Linenoise::AddContinuationMarkers(const char *buf, size_t len, int plen, int cursor_row,
+                                         vector<highlightToken> &tokens) const {
+	std::string result;
+	int rows = 1;
+	int cols = plen;
+	size_t cpos = 0;
+	size_t prev_pos = 0;
+	size_t extra_bytes = 0;    // extra bytes introduced
+	size_t token_position = 0; // token position
+	vector<highlightToken> new_tokens;
+	new_tokens.reserve(tokens.size());
+	while (cpos < len) {
+		bool is_newline = IsNewline(buf[cpos]);
+		NextPosition(buf, len, cpos, rows, cols, plen);
+		for (; prev_pos < cpos; prev_pos++) {
+			result += buf[prev_pos];
+		}
+		if (is_newline) {
+			bool is_cursor_row = rows == cursor_row;
+			const char *prompt = is_cursor_row ? continuationSelectedPrompt : continuationPrompt;
+			if (!continuation_markers) {
+				prompt = "";
+			}
+			size_t continuationLen = strlen(prompt);
+			size_t continuationRender = ComputeRenderWidth(prompt, continuationLen);
+			// pad with spaces prior to prompt
+			for (int i = int(continuationRender); i < plen; i++) {
+				result += " ";
+			}
+			result += prompt;
+			size_t continuationBytes = plen - continuationRender + continuationLen;
+			if (token_position < tokens.size()) {
+				for (; token_position < tokens.size(); token_position++) {
+					if (tokens[token_position].start >= cpos) {
+						// not there yet
+						break;
+					}
+					tokens[token_position].start += extra_bytes;
+					new_tokens.push_back(tokens[token_position]);
+				}
+				tokenType prev_type = tokenType::TOKEN_IDENTIFIER;
+				if (token_position > 0 && token_position < tokens.size() + 1) {
+					prev_type = tokens[token_position - 1].type;
+				}
+				highlightToken token;
+				token.start = cpos + extra_bytes;
+				token.type = is_cursor_row ? tokenType::TOKEN_CONTINUATION_SELECTED : tokenType::TOKEN_CONTINUATION;
+				token.search_match = false;
+				new_tokens.push_back(token);
+
+				token.start = cpos + extra_bytes + continuationBytes;
+				token.type = prev_type;
+				token.search_match = false;
+				new_tokens.push_back(token);
+			}
+			extra_bytes += continuationBytes;
+		}
+	}
+	for (; prev_pos < cpos; prev_pos++) {
+		result += buf[prev_pos];
+	}
+	for (; token_position < tokens.size(); token_position++) {
+		tokens[token_position].start += extra_bytes;
+		new_tokens.push_back(tokens[token_position]);
+	}
+	tokens = std::move(new_tokens);
+	return result;
+}
+
+// insert a token of length 1 of the specified type
+static void InsertToken(tokenType insert_type, idx_t insert_pos, vector<highlightToken> &tokens) {
+	vector<highlightToken> new_tokens;
+	new_tokens.reserve(tokens.size() + 1);
+	idx_t i;
+	bool found = false;
+	for (i = 0; i < tokens.size(); i++) {
+		// find the exact position where we need to insert the token
+		if (tokens[i].start == insert_pos) {
+			// this token is exactly at this render position
+
+			// insert highlighting for the bracket
+			highlightToken token;
+			token.start = insert_pos;
+			token.type = insert_type;
+			token.search_match = false;
+			new_tokens.push_back(token);
+
+			// now we need to insert the other token ONLY if the other token is not immediately following this one
+			if (i + 1 >= tokens.size() || tokens[i + 1].start > insert_pos + 1) {
+				token.start = insert_pos + 1;
+				token.type = tokens[i].type;
+				token.search_match = false;
+				new_tokens.push_back(token);
+			}
+			i++;
+			found = true;
+			break;
+		} else if (tokens[i].start > insert_pos) {
+			// the next token is AFTER the render position
+			// insert highlighting for the bracket
+			highlightToken token;
+			token.start = insert_pos;
+			token.type = insert_type;
+			token.search_match = false;
+			new_tokens.push_back(token);
+
+			// now just insert the next token
+			new_tokens.push_back(tokens[i]);
+			i++;
+			found = true;
+			break;
+		} else {
+			// insert the token
+			new_tokens.push_back(tokens[i]);
+		}
+	}
+	// copy over the remaining tokens
+	for (; i < tokens.size(); i++) {
+		new_tokens.push_back(tokens[i]);
+	}
+	if (!found) {
+		// token was not added - add it to the end
+		highlightToken token;
+		token.start = insert_pos;
+		token.type = insert_type;
+		token.search_match = false;
+		new_tokens.push_back(token);
+	}
+	tokens = std::move(new_tokens);
+}
+
+enum class ScanState { STANDARD, IN_SINGLE_QUOTE, IN_DOUBLE_QUOTE, IN_COMMENT, DOLLAR_QUOTED_STRING };
+
+static void OpenBracket(vector<idx_t> &brackets, vector<idx_t> &cursor_brackets, idx_t pos, idx_t i) {
+	// check if the cursor is at this position
+	if (pos == i) {
+		// cursor is exactly on this position - always highlight this bracket
+		if (!cursor_brackets.empty()) {
+			cursor_brackets.clear();
+		}
+		cursor_brackets.push_back(i);
+	}
+	if (cursor_brackets.empty() && ((i + 1) == pos || (pos + 1) == i)) {
+		// cursor is either BEFORE or AFTER this bracket and we don't have any highlighted bracket yet
+		// highlight this bracket
+		cursor_brackets.push_back(i);
+	}
+	brackets.push_back(i);
+}
+
+static void CloseBracket(vector<idx_t> &brackets, vector<idx_t> &cursor_brackets, idx_t pos, idx_t i,
+                         vector<idx_t> &errors) {
+	if (pos == i) {
+		// cursor is on this closing bracket
+		// clear any selected brackets - we always select this one
+		cursor_brackets.clear();
+	}
+	if (brackets.empty()) {
+		// closing bracket without matching opening bracket
+		errors.push_back(i);
+	} else {
+		if (cursor_brackets.size() == 1) {
+			if (cursor_brackets.back() == brackets.back()) {
+				// this closing bracket matches the highlighted opening cursor bracket - highlight both
+				cursor_brackets.push_back(i);
+			}
+		} else if (cursor_brackets.empty() && (pos == i || (i + 1) == pos || (pos + 1) == i)) {
+			// no cursor bracket selected yet and cursor is BEFORE or AFTER this bracket
+			// add this bracket
+			cursor_brackets.push_back(i);
+			cursor_brackets.push_back(brackets.back());
+		}
+		brackets.pop_back();
+	}
+}
+
+static void HandleBracketErrors(const vector<idx_t> &brackets, vector<idx_t> &errors) {
+	if (brackets.empty()) {
+		return;
+	}
+	// if there are unclosed brackets remaining not all brackets were closed
+	for (auto &bracket : brackets) {
+		errors.push_back(bracket);
+	}
+}
+
+void Linenoise::AddErrorHighlighting(idx_t render_start, idx_t render_end, vector<highlightToken> &tokens) const {
+	static constexpr const idx_t MAX_ERROR_LENGTH = 2000;
+	if (!enableErrorRendering) {
+		return;
+	}
+	if (len >= MAX_ERROR_LENGTH) {
+		return;
+	}
+	// do a pass over the buffer to collect errors:
+	// * brackets without matching closing/opening bracket
+	// * single quotes without matching closing single quote
+	// * double quote without matching double quote
+	ScanState state = ScanState::STANDARD;
+	vector<idx_t> brackets;        // ()
+	vector<idx_t> square_brackets; // []
+	vector<idx_t> curly_brackets;  // {}
+	vector<idx_t> errors;
+	vector<idx_t> cursor_brackets;
+	vector<idx_t> comment_start;
+	vector<idx_t> comment_end;
+	string dollar_quote_marker;
+	idx_t quote_pos = 0;
+	for (idx_t i = 0; i < len; i++) {
+		auto c = buf[i];
+		switch (state) {
+		case ScanState::STANDARD:
+			switch (c) {
+			case '-':
+				if (i + 1 < len && buf[i + 1] == '-') {
+					// -- puts us in a comment
+					comment_start.push_back(i);
+					i++;
+					state = ScanState::IN_COMMENT;
+					break;
+				}
+				break;
+			case '\'':
+				state = ScanState::IN_SINGLE_QUOTE;
+				quote_pos = i;
+				break;
+			case '\"':
+				state = ScanState::IN_DOUBLE_QUOTE;
+				quote_pos = i;
+				break;
+			case '(':
+				OpenBracket(brackets, cursor_brackets, pos, i);
+				break;
+			case '[':
+				OpenBracket(square_brackets, cursor_brackets, pos, i);
+				break;
+			case '{':
+				OpenBracket(curly_brackets, cursor_brackets, pos, i);
+				break;
+			case ')':
+				CloseBracket(brackets, cursor_brackets, pos, i, errors);
+				break;
+			case ']':
+				CloseBracket(square_brackets, cursor_brackets, pos, i, errors);
+				break;
+			case '}':
+				CloseBracket(curly_brackets, cursor_brackets, pos, i, errors);
+				break;
+			case '$': { // dollar symbol
+				if (i + 1 >= len) {
+					// we need more than just a dollar
+					break;
+				}
+				// check if this is a dollar-quoted string
+				idx_t next_dollar = 0;
+				for (idx_t idx = i + 1; idx < len; idx++) {
+					if (buf[idx] == '$') {
+						// found the next dollar
+						next_dollar = idx;
+						break;
+					}
+					// all characters can be between A-Z, a-z or \200 - \377
+					if (buf[idx] >= 'A' && buf[idx] <= 'Z') {
+						continue;
+					}
+					if (buf[idx] >= 'a' && buf[idx] <= 'z') {
+						continue;
+					}
+					if (buf[idx] >= '\200' && buf[idx] <= '\377') {
+						continue;
+					}
+					// the first character CANNOT be a numeric, only subsequent characters
+					if (idx > i + 1 && buf[idx] >= '0' && buf[idx] <= '9') {
+						continue;
+					}
+					// not a dollar quoted string
+					break;
+				}
+				if (next_dollar == 0) {
+					// not a dollar quoted string
+					break;
+				}
+				// dollar quoted string
+				state = ScanState::DOLLAR_QUOTED_STRING;
+				quote_pos = i;
+				i = next_dollar;
+				if (i < len) {
+					// found a complete marker - store it
+					idx_t marker_start = quote_pos + 1;
+					dollar_quote_marker = string(buf + marker_start, i - marker_start);
+				}
+				break;
+			}
+			default:
+				break;
+			}
+			break;
+		case ScanState::IN_COMMENT:
+			// comment state - the only thing that will get us out is a newline
+			switch (c) {
+			case '\r':
+			case '\n':
+				// newline - left comment state
+				state = ScanState::STANDARD;
+				comment_end.push_back(i);
+				break;
+			default:
+				break;
+			}
+			break;
+		case ScanState::IN_SINGLE_QUOTE:
+			// single quote - all that will get us out is an unescaped single-quote
+			if (c == '\'') {
+				if (i + 1 < len && buf[i + 1] == '\'') {
+					// double single-quote means the quote is escaped - continue
+					i++;
+					break;
+				} else {
+					// otherwise revert to standard scan state
+					state = ScanState::STANDARD;
+					break;
+				}
+			}
+			break;
+		case ScanState::IN_DOUBLE_QUOTE:
+			// double quote - all that will get us out is an unescaped quote
+			if (c == '"') {
+				if (i + 1 < len && buf[i + 1] == '"') {
+					// double quote means the quote is escaped - continue
+					i++;
+					break;
+				} else {
+					// otherwise revert to standard scan state
+					state = ScanState::STANDARD;
+					break;
+				}
+			}
+			break;
+		case ScanState::DOLLAR_QUOTED_STRING: {
+			// dollar-quoted string - all that will get us out is a $[marker]$
+			if (c != '$') {
+				break;
+			}
+			if (i + 1 >= len) {
+				// no room for the final dollar
+				break;
+			}
+			// skip to the next dollar symbol
+			idx_t start = i + 1;
+			idx_t end = start;
+			while (end < len && buf[end] != '$') {
+				end++;
+			}
+			if (end >= len) {
+				// no final dollar found - continue as normal
+				break;
+			}
+			if (end - start != dollar_quote_marker.size()) {
+				// length mismatch - cannot match
+				break;
+			}
+			if (memcmp(buf + start, dollar_quote_marker.c_str(), dollar_quote_marker.size()) != 0) {
+				// marker mismatch
+				break;
+			}
+			// marker found! revert to standard state
+			dollar_quote_marker = string();
+			state = ScanState::STANDARD;
+			i = end;
+			break;
+		}
+		default:
+			break;
+		}
+	}
+	if (state == ScanState::IN_DOUBLE_QUOTE || state == ScanState::IN_SINGLE_QUOTE ||
+	    state == ScanState::DOLLAR_QUOTED_STRING) {
+		// quote is never closed
+		errors.push_back(quote_pos);
+	}
+	HandleBracketErrors(brackets, errors);
+	HandleBracketErrors(square_brackets, errors);
+	HandleBracketErrors(curly_brackets, errors);
+
+	// insert all the errors for highlighting
+	for (auto &error : errors) {
+		Linenoise::Log("Error found at position %llu\n", error);
+		if (error < render_start || error > render_end) {
+			continue;
+		}
+		auto render_error = error - render_start;
+		InsertToken(tokenType::TOKEN_ERROR, render_error, tokens);
+	}
+	if (cursor_brackets.size() != 2) {
+		// no matching cursor brackets found
+		cursor_brackets.clear();
+	}
+	// insert bracket for highlighting
+	for (auto &bracket_position : cursor_brackets) {
+		Linenoise::Log("Highlight bracket at position %d\n", bracket_position);
+		if (bracket_position < render_start || bracket_position > render_end) {
+			continue;
+		}
+
+		idx_t render_position = bracket_position - render_start;
+		InsertToken(tokenType::TOKEN_BRACKET, render_position, tokens);
+	}
+	// insert comments
+	if (!comment_start.empty()) {
+		vector<highlightToken> new_tokens;
+		new_tokens.reserve(tokens.size());
+		idx_t token_idx = 0;
+		for (idx_t c = 0; c < comment_start.size(); c++) {
+			auto c_start = comment_start[c];
+			auto c_end = c < comment_end.size() ? comment_end[c] : len;
+			if (c_start < render_start || c_end > render_end) {
+				continue;
+			}
+			Linenoise::Log("Comment at position %d to %d\n", c_start, c_end);
+			c_start -= render_start;
+			c_end -= render_start;
+			bool inserted_comment = false;
+
+			highlightToken comment_token;
+			comment_token.start = c_start;
+			comment_token.type = tokenType::TOKEN_COMMENT;
+			comment_token.search_match = false;
+
+			for (; token_idx < tokens.size(); token_idx++) {
+				if (tokens[token_idx].start >= c_start) {
+					// insert the comment here
+					new_tokens.push_back(comment_token);
+					inserted_comment = true;
+					break;
+				}
+				new_tokens.push_back(tokens[token_idx]);
+			}
+			if (!inserted_comment) {
+				new_tokens.push_back(comment_token);
+			} else {
+				// skip all tokens until we exit the comment again
+				for (; token_idx < tokens.size(); token_idx++) {
+					if (tokens[token_idx].start > c_end) {
+						break;
+					}
+				}
+			}
+		}
+		for (; token_idx < tokens.size(); token_idx++) {
+			new_tokens.push_back(tokens[token_idx]);
+		}
+		tokens = std::move(new_tokens);
+	}
+}
+
+static bool IsCompletionCharacter(char c) {
+	if (c >= 'A' && c <= 'Z') {
+		return true;
+	}
+	if (c >= 'a' && c <= 'z') {
+		return true;
+	}
+	if (c == '_') {
+		return true;
+	}
+	return false;
+}
+
+bool Linenoise::AddCompletionMarker(const char *buf, idx_t len, string &result_buffer,
+                                    vector<highlightToken> &tokens) const {
+	if (!enableCompletionRendering) {
+		return false;
+	}
+	if (!continuation_markers) {
+		// don't render when pressing ctrl+c, only when editing
+		return false;
+	}
+	static constexpr const idx_t MAX_COMPLETION_LENGTH = 1000;
+	if (len >= MAX_COMPLETION_LENGTH) {
+		return false;
+	}
+	if (!insert || pos != len) {
+		// only show when inserting a character at the end
+		return false;
+	}
+	if (pos < 3) {
+		// we need at least 3 bytes
+		return false;
+	}
+	if (!tokens.empty() && tokens.back().type == tokenType::TOKEN_ERROR) {
+		// don't show auto-completion when we have errors
+		return false;
+	}
+	// we ONLY show completion if we have typed at least three characters that are supported for completion
+	// for now this is ONLY the characters a-z, A-Z and underscore (_)
+	for (idx_t i = pos - 3; i < pos; i++) {
+		if (!IsCompletionCharacter(buf[i])) {
+			return false;
+		}
+	}
+	auto completion = TabComplete();
+	if (completion.completions.empty()) {
+		// no completions found
+		return false;
+	}
+	if (completion.completions[0].completion.size() <= len) {
+		// completion is not long enough
+		return false;
+	}
+	// we have stricter requirements for rendering completions - the completion must match exactly
+	for (idx_t i = pos; i > 0; i--) {
+		auto cpos = i - 1;
+		if (!IsCompletionCharacter(buf[cpos])) {
+			break;
+		}
+		if (completion.completions[0].completion[cpos] != buf[cpos]) {
+			return false;
+		}
+	}
+	// add the first completion found for rendering purposes
+	result_buffer = string(buf, len);
+	result_buffer += completion.completions[0].completion.substr(len);
+
+	highlightToken completion_token;
+	completion_token.start = len;
+	completion_token.type = tokenType::TOKEN_COMMENT;
+	completion_token.search_match = true;
+	tokens.push_back(completion_token);
+	return true;
+}
+
+/* Multi line low level line refresh.
+ *
+ * Rewrite the currently edited line accordingly to the buffer content,
+ * cursor position, and number of columns of the terminal. */
+void Linenoise::RefreshMultiLine() {
+	if (!render) {
+		return;
+	}
+	char seq[64];
+	int plen = GetPromptWidth();
+	// utf8 in prompt, get render width
+	int rows, cols;
+	int new_cursor_row, new_cursor_x;
+	PositionToColAndRow(pos, new_cursor_row, new_cursor_x, rows, cols);
+	int col; /* column position, zero-based. */
+	int old_rows = maxrows ? maxrows : 1;
+	int fd = ofd;
+	std::string highlight_buffer;
+	auto render_buf = this->buf;
+	auto render_len = this->len;
+	idx_t render_start = 0;
+	idx_t render_end = render_len;
+	if (clear_screen) {
+		old_cursor_rows = 0;
+		old_rows = 0;
+		clear_screen = false;
+	}
+	if (rows > ws.ws_row) {
+		// the text does not fit in the terminal (too many rows)
+		// enable scrolling mode
+		// check if, given the current y_scroll, the cursor is visible
+		// display range is [y_scroll, y_scroll + ws.ws_row]
+		if (new_cursor_row < int(y_scroll) + 1) {
+			y_scroll = new_cursor_row - 1;
+		} else if (new_cursor_row > int(y_scroll) + int(ws.ws_row)) {
+			y_scroll = new_cursor_row - ws.ws_row;
+		}
+		// display only characters up to the current scroll position
+		if (y_scroll == 0) {
+			render_start = 0;
+		} else {
+			render_start = ColAndRowToPosition(y_scroll, 0);
+		}
+		if (int(y_scroll) + int(ws.ws_row) >= rows) {
+			render_end = len;
+		} else {
+			render_end = ColAndRowToPosition(y_scroll + ws.ws_row, 99999);
+		}
+		new_cursor_row -= y_scroll;
+		render_buf += render_start;
+		render_len = render_end - render_start;
+		Linenoise::Log("truncate to rows %d - %d (render bytes %d to %d)", y_scroll, y_scroll + ws.ws_row, render_start,
+		               render_end);
+		rows = ws.ws_row;
+	} else {
+		y_scroll = 0;
+	}
+
+	/* Update maxrows if needed. */
+	if (rows > (int)maxrows) {
+		maxrows = rows;
+	}
+
+	vector<highlightToken> tokens;
+	if (Highlighting::IsEnabled()) {
+		bool is_dot_command = buf[0] == '.';
+		auto match = search_index < search_matches.size() ? &search_matches[search_index] : nullptr;
+		tokens = Highlighting::Tokenize(render_buf, render_len, is_dot_command, match);
+
+		// add error highlighting
+		AddErrorHighlighting(render_start, render_end, tokens);
+
+		// add completion hint
+		if (AddCompletionMarker(render_buf, render_len, highlight_buffer, tokens)) {
+			render_buf = (char *)highlight_buffer.c_str();
+			render_len = highlight_buffer.size();
+		}
+	}
+	if (rows > 1) {
+		// add continuation markers
+		highlight_buffer = AddContinuationMarkers(render_buf, render_len, plen,
+		                                          y_scroll > 0 ? new_cursor_row + 1 : new_cursor_row, tokens);
+		render_buf = (char *)highlight_buffer.c_str();
+		render_len = highlight_buffer.size();
+	}
+	if (duckdb::Utf8Proc::IsValid(render_buf, render_len)) {
+		if (Highlighting::IsEnabled()) {
+			highlight_buffer = Highlighting::HighlightText(render_buf, render_len, 0, render_len, tokens);
+			render_buf = (char *)highlight_buffer.c_str();
+			render_len = highlight_buffer.size();
+		}
+	}
+
+	/* First step: clear all the lines used before. To do so start by
+	 * going to the last row. */
+	AppendBuffer append_buffer;
+	if (old_rows - old_cursor_rows > 0) {
+		Linenoise::Log("go down %d", old_rows - old_cursor_rows);
+		snprintf(seq, 64, "\x1b[%dB", old_rows - int(old_cursor_rows));
+		append_buffer.Append(seq);
+	}
+
+	/* Now for every row clear it, go up. */
+	for (int j = 0; j < old_rows - 1; j++) {
+		Linenoise::Log("clear+up");
+		append_buffer.Append("\r\x1b[0K\x1b[1A");
+	}
+
+	/* Clean the top line. */
+	Linenoise::Log("clear");
+	append_buffer.Append("\r\x1b[0K");
+
+	/* Write the prompt and the current buffer content */
+	if (y_scroll == 0) {
+		append_buffer.Append(prompt);
+	}
+	append_buffer.Append(render_buf, render_len);
+
+	/* Show hints if any. */
+	RefreshShowHints(append_buffer, plen);
+
+	/* If we are at the very end of the screen with our prompt, we need to
+	 * emit a newline and move the prompt to the first column. */
+	Linenoise::Log("pos > 0 %d", pos > 0 ? 1 : 0);
+	Linenoise::Log("pos == len %d", pos == len ? 1 : 0);
+	Linenoise::Log("new_cursor_x == cols %d", new_cursor_x == ws.ws_col ? 1 : 0);
+	if (pos > 0 && pos == len && new_cursor_x == ws.ws_col) {
+		Linenoise::Log("<newline>", 0);
+		append_buffer.Append("\n");
+		append_buffer.Append("\r");
+		rows++;
+		new_cursor_row++;
+		new_cursor_x = 0;
+		if (rows > (int)maxrows) {
+			maxrows = rows;
+		}
+	}
+	Linenoise::Log("render %d rows (old rows %d)", rows, old_rows);
+
+	/* Move cursor to right position. */
+	Linenoise::Log("new_cursor_row %d", new_cursor_row);
+	Linenoise::Log("new_cursor_x %d", new_cursor_x);
+	Linenoise::Log("len %d", len);
+	Linenoise::Log("old_cursor_rows %d", old_cursor_rows);
+	Linenoise::Log("pos %d", pos);
+	Linenoise::Log("max cols %d", ws.ws_col);
+
+	/* Go up till we reach the expected position. */
+	if (rows - new_cursor_row > 0) {
+		Linenoise::Log("go-up %d", rows - new_cursor_row);
+		snprintf(seq, 64, "\x1b[%dA", rows - new_cursor_row);
+		append_buffer.Append(seq);
+	}
+
+	/* Set column. */
+	col = new_cursor_x;
+	Linenoise::Log("set col %d", 1 + col);
+	if (col) {
+		snprintf(seq, 64, "\r\x1b[%dC", col);
+	} else {
+		snprintf(seq, 64, "\r");
+	}
+	append_buffer.Append(seq);
+
+	Linenoise::Log("\n");
+	old_cursor_rows = new_cursor_row;
+	append_buffer.Write(fd);
+}
+
+/* Calls the two low level functions refreshSingleLine() or
+ * refreshMultiLine() according to the selected mode. */
+void Linenoise::RefreshLine() {
+	if (Terminal::IsMultiline()) {
+		RefreshMultiLine();
+	} else {
+		RefreshSingleLine();
+	}
+}
+
+} // namespace duckdb

external/duckdb/tools/shell/linenoise/terminal.cpp

@@ -0,0 +1,496 @@
+#include "terminal.hpp"
+#include "history.hpp"
+#include "linenoise.hpp"
+#include <termios.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <sys/ioctl.h>
+#include <sys/select.h>
+#include <sys/time.h>
+
+namespace duckdb {
+
+static int mlmode = 1;              /* Multi line mode. Default is multi line. */
+static struct termios orig_termios; /* In order to restore at exit.*/
+static int atexit_registered = 0;   /* Register atexit just 1 time. */
+static int rawmode = 0;             /* For atexit() function to check if restore is needed*/
+static const char *unsupported_term[] = {"dumb", "cons25", "emacs", NULL};
+
+/* At exit we'll try to fix the terminal to the initial conditions. */
+static void linenoiseAtExit(void) {
+	Terminal::DisableRawMode();
+	History::Free();
+}
+
+/* Return true if the terminal name is in the list of terminals we know are
+ * not able to understand basic escape sequences. */
+int Terminal::IsUnsupportedTerm() {
+	char *term = getenv("TERM");
+	int j;
+
+	if (!term) {
+		return 0;
+	}
+	for (j = 0; unsupported_term[j]; j++) {
+		if (!strcasecmp(term, unsupported_term[j])) {
+			return 1;
+		}
+	}
+	return 0;
+}
+
+/* Raw mode: 1960 magic shit. */
+int Terminal::EnableRawMode() {
+	int fd = STDIN_FILENO;
+
+	if (!isatty(STDIN_FILENO)) {
+		errno = ENOTTY;
+		return -1;
+	}
+	if (!atexit_registered) {
+		atexit(linenoiseAtExit);
+		atexit_registered = 1;
+	}
+	if (tcgetattr(fd, &orig_termios) == -1) {
+		errno = ENOTTY;
+		return -1;
+	}
+
+	auto raw = orig_termios; /* modify the original mode */
+	/* input modes: no break, no CR to NL, no parity check, no strip char,
+	 * no start/stop output control. */
+	raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON);
+	/* output modes - disable post processing */
+	raw.c_oflag &= ~(OPOST);
+#ifdef IUTF8
+	/* control modes - set 8 bit chars */
+	raw.c_iflag |= IUTF8;
+#endif
+	raw.c_cflag |= CS8;
+	/* local modes - choing off, canonical off, no extended functions,
+	 * no signal chars (^Z,^C) */
+	raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG);
+	/* control chars - set return condition: min number of bytes and timer.
+	 * We want read to return every single byte, without timeout. */
+	raw.c_cc[VMIN] = 1;
+	raw.c_cc[VTIME] = 0; /* 1 byte, no timer */
+
+	/* put terminal in raw mode after flushing */
+	if (tcsetattr(fd, TCSADRAIN, &raw) < 0) {
+		errno = ENOTTY;
+		return -1;
+	}
+	rawmode = 1;
+	return 0;
+}
+
+void Terminal::DisableRawMode() {
+	int fd = STDIN_FILENO;
+	/* Don't even check the return value as it's too late. */
+	if (rawmode && tcsetattr(fd, TCSADRAIN, &orig_termios) != -1) {
+		rawmode = 0;
+	}
+}
+
+bool Terminal::IsMultiline() {
+	return mlmode;
+}
+
+bool Terminal::IsAtty() {
+	return isatty(STDIN_FILENO);
+}
+
+/* This function is called when linenoise() is called with the standard
+ * input file descriptor not attached to a TTY. So for example when the
+ * program using linenoise is called in pipe or with a file redirected
+ * to its standard input. In this case, we want to be able to return the
+ * line regardless of its length (by default we are limited to 4k). */
+char *Terminal::EditNoTTY() {
+	char *line = NULL;
+	size_t len = 0, maxlen = 0;
+
+	while (1) {
+		if (len == maxlen) {
+			if (maxlen == 0)
+				maxlen = 16;
+			maxlen *= 2;
+			char *oldval = line;
+			line = (char *)realloc(line, maxlen);
+			if (line == NULL) {
+				if (oldval)
+					free(oldval);
+				return NULL;
+			}
+		}
+		int c = fgetc(stdin);
+		if (c == EOF || c == '\n') {
+			if (c == EOF && len == 0) {
+				free(line);
+				return NULL;
+			} else {
+				line[len] = '\0';
+				return line;
+			}
+		} else {
+			line[len] = c;
+			len++;
+		}
+	}
+}
+
+/* This function calls the line editing function linenoiseEdit() using
+ * the STDIN file descriptor set in raw mode. */
+int Terminal::EditRaw(char *buf, size_t buflen, const char *prompt) {
+	int count;
+
+	if (buflen == 0) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	if (Terminal::EnableRawMode() == -1) {
+		return -1;
+	}
+	Linenoise l(STDIN_FILENO, STDOUT_FILENO, buf, buflen, prompt);
+	count = l.Edit();
+	Terminal::DisableRawMode();
+	printf("\n");
+	return count;
+}
+
+// returns true if there is more data available to read in a particular stream
+int Terminal::HasMoreData(int fd) {
+	fd_set rfds;
+	FD_ZERO(&rfds);
+	FD_SET(fd, &rfds);
+
+	// no timeout: return immediately
+	struct timeval tv;
+	tv.tv_sec = 0;
+	tv.tv_usec = 0;
+	return select(1, &rfds, NULL, NULL, &tv);
+}
+
+/* ======================= Low level terminal handling ====================== */
+
+/* Set if to use or not the multi line mode. */
+void Terminal::SetMultiLine(int ml) {
+	mlmode = ml;
+}
+
+static int parseInt(const char *s, int *offset = nullptr) {
+	int result = 0;
+	int idx;
+	for (idx = 0; s[idx]; idx++) {
+		char c = s[idx];
+		if (c < '0' || c > '9') {
+			break;
+		}
+		result = result * 10 + c - '0';
+		if (result > 1000000) {
+			result = 1000000;
+		}
+	}
+	if (offset) {
+		*offset = idx;
+	}
+	return result;
+}
+
+static int tryParseEnv(const char *env_var) {
+	char *s;
+	s = getenv(env_var);
+	if (!s) {
+		return 0;
+	}
+	return parseInt(s);
+}
+
+/* Use the ESC [6n escape sequence to query the cursor position
+ * and return it. On error -1 is returned, on success the position of the
+ * cursor. */
+TerminalSize Terminal::GetCursorPosition() {
+	int ifd = STDIN_FILENO;
+	int ofd = STDOUT_FILENO;
+	TerminalSize ws;
+
+	char buf[32];
+	unsigned int i = 0;
+
+	/* Report cursor location */
+	if (write(ofd, "\x1b[6n", 4) != 4) {
+		return ws;
+	}
+
+	/* Read the response: ESC [ rows ; cols R */
+	while (i < sizeof(buf) - 1) {
+		if (read(ifd, buf + i, 1) != 1) {
+			break;
+		}
+		if (buf[i] == 'R') {
+			break;
+		}
+		i++;
+	}
+	buf[i] = '\0';
+
+	/* Parse it. */
+	if (buf[0] != ESC || buf[1] != '[') {
+		return ws;
+	}
+	int offset = 2;
+	int new_offset;
+	ws.ws_row = parseInt(buf + offset, &new_offset);
+	offset += new_offset;
+	if (buf[offset] != ';') {
+		return ws;
+	}
+	offset++;
+	ws.ws_col = parseInt(buf + offset);
+	return ws;
+}
+
+TerminalSize Terminal::TryMeasureTerminalSize() {
+	int ofd = STDOUT_FILENO;
+	/* ioctl() failed. Try to query the terminal itself. */
+	TerminalSize start, result;
+
+	/* Get the initial position so we can restore it later. */
+	start = GetCursorPosition();
+	if (!start.ws_col) {
+		return result;
+	}
+
+	/* Go to bottom-right margin */
+	if (write(ofd, "\x1b[999;999f", 10) != 10) {
+		return result;
+	}
+	result = GetCursorPosition();
+	if (!result.ws_col) {
+		return result;
+	}
+
+	/* Restore position. */
+	char seq[32];
+	snprintf(seq, 32, "\x1b[%d;%df", start.ws_row, start.ws_col);
+	if (write(ofd, seq, strlen(seq)) == -1) {
+		/* Can't recover... */
+	}
+	return result;
+}
+
+/* Try to get the number of columns in the current terminal, or assume 80
+ * if it fails. */
+TerminalSize Terminal::GetTerminalSize() {
+	TerminalSize result;
+
+	// try ioctl first
+	{
+		struct winsize ws;
+		ioctl(1, TIOCGWINSZ, &ws);
+		result.ws_col = ws.ws_col;
+		result.ws_row = ws.ws_row;
+	}
+	// try ROWS and COLUMNS env variables
+	if (!result.ws_col) {
+		result.ws_col = tryParseEnv("COLUMNS");
+	}
+	if (!result.ws_row) {
+		result.ws_row = tryParseEnv("ROWS");
+	}
+	// if those fail measure the size by moving the cursor to the corner and fetching the position
+	if (!result.ws_col || !result.ws_row) {
+		TerminalSize measured_size = TryMeasureTerminalSize();
+		Linenoise::Log("measured size col %d,row %d -- ", measured_size.ws_row, measured_size.ws_col);
+		if (measured_size.ws_row) {
+			result.ws_row = measured_size.ws_row;
+		}
+		if (measured_size.ws_col) {
+			result.ws_col = measured_size.ws_col;
+		}
+	}
+	// if all else fails use defaults (80,24)
+	if (!result.ws_col) {
+		result.ws_col = 80;
+	}
+	if (!result.ws_row) {
+		result.ws_row = 24;
+	}
+	return result;
+}
+
+/* Clear the screen. Used to handle ctrl+l */
+void Terminal::ClearScreen() {
+	if (write(STDOUT_FILENO, "\x1b[H\x1b[2J", 7) <= 0) {
+		/* nothing to do, just to avoid warning. */
+	}
+}
+
+/* Beep, used for completion when there is nothing to complete or when all
+ * the choices were already shown. */
+void Terminal::Beep() {
+	fprintf(stderr, "\x7");
+	fflush(stderr);
+}
+
+EscapeSequence Terminal::ReadEscapeSequence(int ifd) {
+	char seq[5];
+	idx_t length = ReadEscapeSequence(ifd, seq);
+	if (length == 0) {
+		return EscapeSequence::INVALID;
+	}
+	Linenoise::Log("escape of length %d\n", length);
+	switch (length) {
+	case 1:
+		if (seq[0] >= 'a' && seq[0] <= 'z') {
+			return EscapeSequence(idx_t(EscapeSequence::ALT_A) + (seq[0] - 'a'));
+		}
+		if (seq[0] >= 'A' && seq[0] <= 'Z') {
+			return EscapeSequence(idx_t(EscapeSequence::ALT_A) + (seq[0] - 'A'));
+		}
+		switch (seq[0]) {
+		case BACKSPACE:
+			return EscapeSequence::ALT_BACKSPACE;
+		case ESC:
+			return EscapeSequence::ESCAPE;
+		case '<':
+			return EscapeSequence::ALT_LEFT_ARROW;
+		case '>':
+			return EscapeSequence::ALT_RIGHT_ARROW;
+		case '\\':
+			return EscapeSequence::ALT_BACKSLASH;
+		default:
+			Linenoise::Log("unrecognized escape sequence of length 1 - %d\n", seq[0]);
+			break;
+		}
+		break;
+	case 2:
+		if (seq[0] == 'O') {
+			switch (seq[1]) {
+			case 'A': /* Up */
+				return EscapeSequence::UP;
+			case 'B': /* Down */
+				return EscapeSequence::DOWN;
+			case 'C': /* Right */
+				return EscapeSequence::RIGHT;
+			case 'D': /* Left */
+				return EscapeSequence::LEFT;
+			case 'H': /* Home */
+				return EscapeSequence::HOME;
+			case 'F': /* End*/
+				return EscapeSequence::END;
+			case 'c':
+				return EscapeSequence::ALT_F;
+			case 'd':
+				return EscapeSequence::ALT_B;
+			default:
+				Linenoise::Log("unrecognized escape sequence (O) %d\n", seq[1]);
+				break;
+			}
+		} else if (seq[0] == '[') {
+			switch (seq[1]) {
+			case 'A': /* Up */
+				return EscapeSequence::UP;
+			case 'B': /* Down */
+				return EscapeSequence::DOWN;
+			case 'C': /* Right */
+				return EscapeSequence::RIGHT;
+			case 'D': /* Left */
+				return EscapeSequence::LEFT;
+			case 'H': /* Home */
+				return EscapeSequence::HOME;
+			case 'F': /* End*/
+				return EscapeSequence::END;
+			case 'Z': /* Shift Tab */
+				return EscapeSequence::SHIFT_TAB;
+			default:
+				Linenoise::Log("unrecognized escape sequence (seq[1]) %d\n", seq[1]);
+				break;
+			}
+		} else {
+			Linenoise::Log("unrecognized escape sequence of length %d (%d %d)\n", length, seq[0], seq[1]);
+		}
+		break;
+	case 3:
+		if (seq[2] == '~') {
+			switch (seq[1]) {
+			case '1':
+				return EscapeSequence::HOME;
+			case '3': /* Delete key. */
+				return EscapeSequence::DELETE;
+			case '4':
+			case '8':
+				return EscapeSequence::END;
+			default:
+				Linenoise::Log("unrecognized escape sequence (~) %d\n", seq[1]);
+				break;
+			}
+		} else if (seq[1] == '5' && seq[2] == 'C') {
+			return EscapeSequence::ALT_F;
+		} else if (seq[1] == '5' && seq[2] == 'D') {
+			return EscapeSequence::ALT_B;
+		} else {
+			Linenoise::Log("unrecognized escape sequence of length %d\n", length);
+		}
+		break;
+	case 5:
+		if (memcmp(seq, "[1;5C", 5) == 0 || memcmp(seq, "[1;3C", 5) == 0) {
+			// [1;5C: move word right
+			return EscapeSequence::CTRL_MOVE_FORWARDS;
+		} else if (memcmp(seq, "[1;5D", 5) == 0 || memcmp(seq, "[1;3D", 5) == 0) {
+			// [1;5D: move word left
+			return EscapeSequence::CTRL_MOVE_BACKWARDS;
+		} else {
+			Linenoise::Log("unrecognized escape sequence (;) %d\n", seq[1]);
+		}
+		break;
+	default:
+		Linenoise::Log("unrecognized escape sequence of length %d\n", length);
+		break;
+	}
+	return EscapeSequence::UNKNOWN;
+}
+
+idx_t Terminal::ReadEscapeSequence(int ifd, char seq[]) {
+	if (read(ifd, seq, 1) == -1) {
+		return 0;
+	}
+	switch (seq[0]) {
+	case 'O':
+	case '[':
+		// these characters have multiple bytes following them
+		break;
+	default:
+		return 1;
+	}
+	if (read(ifd, seq + 1, 1) == -1) {
+		return 0;
+	}
+
+	if (seq[0] != '[') {
+		return 2;
+	}
+	if (seq[1] < '0' || seq[1] > '9') {
+		return 2;
+	}
+	/* Extended escape, read additional byte. */
+	if (read(ifd, seq + 2, 1) == -1) {
+		return 0;
+	}
+	if (seq[2] == ';') {
+		// read 2 extra bytes
+		if (read(ifd, seq + 3, 2) == -1) {
+			return 0;
+		}
+		return 5;
+	} else {
+		return 3;
+	}
+}
+
+} // namespace duckdb