marsultor/email-tracker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

should be it

Browse Source
This commit is contained in:
Mars Ultor
2025-10-24 19:21:19 -05:00
parent a4b23fc57c
commit f09560c7b1
14047 changed files with 3161551 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
external/duckdb/tools/shell/CMakeLists.txt vendored Normal file
View File

@@ -0,0 +1,69 @@
include_directories(include)
include_directories(../sqlite3_api_wrapper/include)
if(NOT WIN32)
add_subdirectory(linenoise)
add_definitions(-DHAVE_LINENOISE=1)
include_directories(../../third_party/utf8proc/include)
include_directories(linenoise/include)
endif()
set(SHELL_SOURCES ${SHELL_SOURCES} shell.cpp shell_renderer.cpp
shell_highlight.cpp)
option(STATIC_LIBCPP "Statically link CLI to libc++" FALSE)
add_executable(shell ${SHELL_SOURCES})
target_link_libraries(shell sqlite3_api_wrapper_static
${DUCKDB_EXTRA_LINK_FLAGS})
link_threads(shell "")
if(STATIC_LIBCPP)
message("Statically linking CLI")
target_link_libraries(shell -static-libstdc++ -static-libgcc)
endif()
if(NOT AMALGAMATION_BUILD AND NOT WIN32)
target_link_libraries(shell duckdb_utf8proc)
endif()
function(ensure_variable_is_number INPUT_VERSION OUT_RESULT)
if(NOT "${${INPUT_VERSION}}" MATCHES "^[0-9]+$")
message(
WARNING
"VERSION PARAMETER ${INPUT_VERSION} \"${${INPUT_VERSION}}\" IS NOT A NUMBER - SETTING TO 0"
)
set(${OUT_RESULT}
0
PARENT_SCOPE)
else()
set(${OUT_RESULT}
${${INPUT_VERSION}}
PARENT_SCOPE)
endif()
endfunction()
if(WIN32 AND NOT MINGW)
string(TIMESTAMP DUCKDB_COPYRIGHT_YEAR "%Y")
ensure_variable_is_number(DUCKDB_MAJOR_VERSION RC_MAJOR_VERSION)
ensure_variable_is_number(DUCKDB_MINOR_VERSION RC_MINOR_VERSION)
ensure_variable_is_number(DUCKDB_PATCH_VERSION RC_PATCH_VERSION)
ensure_variable_is_number(DUCKDB_DEV_ITERATION RC_DEV_ITERATION)
set(CMAKE_RC_FLAGS
"${CMAKE_RC_FLAGS} -D DUCKDB_VERSION=\"${DUCKDB_VERSION}\"")
set(CMAKE_RC_FLAGS
"${CMAKE_RC_FLAGS} -D DUCKDB_MAJOR_VERSION=\"${RC_MAJOR_VERSION}\"")
set(CMAKE_RC_FLAGS
"${CMAKE_RC_FLAGS} -D DUCKDB_MINOR_VERSION=\"${RC_MINOR_VERSION}\"")
set(CMAKE_RC_FLAGS
"${CMAKE_RC_FLAGS} -D DUCKDB_PATCH_VERSION=\"${RC_PATCH_VERSION}\"")
set(CMAKE_RC_FLAGS
"${CMAKE_RC_FLAGS} -D DUCKDB_DEV_ITERATION=\"${RC_DEV_ITERATION}\"")
set(CMAKE_RC_FLAGS
"${CMAKE_RC_FLAGS} -D DUCKDB_COPYRIGHT_YEAR=\"${DUCKDB_COPYRIGHT_YEAR}\"")
target_sources(shell PRIVATE rc/duckdb.rc)
endif()
set_target_properties(shell PROPERTIES OUTPUT_NAME duckdb)
set_target_properties(shell PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${PROJECT_BINARY_DIR})
install(TARGETS shell RUNTIME DESTINATION "${INSTALL_BIN_DIR}")

50
external/duckdb/tools/shell/include/shell_highlight.hpp vendored Normal file
View File

@@ -0,0 +1,50 @@
//===----------------------------------------------------------------------===//
// DuckDB
//
// shell_highlight.hpp
//
//
//===----------------------------------------------------------------------===//
#pragma once
#include "shell_state.hpp"
namespace duckdb_shell {
enum class PrintColor { STANDARD, RED, YELLOW, GREEN, GRAY, BLUE, MAGENTA, CYAN, WHITE };
enum class PrintIntensity { STANDARD, BOLD, UNDERLINE, BOLD_UNDERLINE };
enum class HighlightElementType : uint32_t {
ERROR_TOKEN = 0,
KEYWORD,
NUMERIC_CONSTANT,
STRING_CONSTANT,
LINE_INDICATOR,
COLUMN_NAME,
COLUMN_TYPE,
NUMERIC_VALUE,
STRING_VALUE,
TEMPORAL_VALUE,
NULL_VALUE,
FOOTER,
LAYOUT,
NONE
};
struct ShellHighlight {
explicit ShellHighlight(ShellState &state);
void PrintText(const string &text, PrintOutput output, PrintColor color, PrintIntensity intensity);
void PrintText(const string &text, PrintOutput output, HighlightElementType type);
void PrintError(string error_msg);
bool SetColor(const char *element_type, const char *color, const char *intensity);
public:
ShellState &state;
};
} // namespace duckdb_shell

76
external/duckdb/tools/shell/include/shell_renderer.hpp vendored Normal file
View File

@@ -0,0 +1,76 @@
//===----------------------------------------------------------------------===//
// DuckDB
//
// shell_renderer.hpp
//
//
//===----------------------------------------------------------------------===//
#pragma once
#include "shell_state.hpp"
namespace duckdb_shell {
struct ShellState;
class ShellRenderer {
public:
explicit ShellRenderer(ShellState &state);
virtual ~ShellRenderer() = default;
ShellState &state;
bool show_header;
string col_sep;
string row_sep;
public:
static bool IsColumnar(RenderMode mode);
};
struct ColumnarResult {
idx_t column_count = 0;
vector<string> data;
vector<int> types;
vector<idx_t> column_width;
vector<bool> right_align;
vector<const char *> type_names;
};
struct RowResult {
vector<const char *> column_names;
vector<const char *> data;
vector<int> types;
sqlite3_stmt *pStmt = nullptr;
};
class ColumnRenderer : public ShellRenderer {
public:
explicit ColumnRenderer(ShellState &state);
virtual void RenderHeader(ColumnarResult &result) = 0;
virtual void RenderFooter(ColumnarResult &result);
virtual const char *GetColumnSeparator() = 0;
virtual const char *GetRowSeparator() = 0;
virtual const char *GetRowStart() {
return nullptr;
}
void RenderAlignedValue(ColumnarResult &result, idx_t i);
};
class RowRenderer : public ShellRenderer {
public:
explicit RowRenderer(ShellState &state);
bool first_row = true;
public:
virtual void Render(RowResult &result);
virtual void RenderHeader(RowResult &result);
virtual void RenderRow(RowResult &result) = 0;
virtual void RenderFooter(RowResult &result);
};
} // namespace duckdb_shell

215
external/duckdb/tools/shell/include/shell_state.hpp vendored Normal file
View File

@@ -0,0 +1,215 @@
//===----------------------------------------------------------------------===//
// DuckDB
//
// shell_state.hpp
//
//
//===----------------------------------------------------------------------===//
#pragma once
#include <vector>
#include <string>
#include <cstdint>
#include <memory>
#include "duckdb/common/string_util.hpp"
#include "duckdb/common/unique_ptr.hpp"
struct sqlite3;
struct sqlite3_stmt;
enum class MetadataResult : uint8_t;
namespace duckdb_shell {
using duckdb::unique_ptr;
using std::string;
using std::vector;
struct ColumnarResult;
struct RowResult;
class ColumnRenderer;
class RowRenderer;
using idx_t = uint64_t;
enum class RenderMode : uint32_t {
LINE = 0, /* One column per line. Blank line between records */
COLUMN, /* One record per line in neat columns */
LIST, /* One record per line with a separator */
SEMI, /* Same as RenderMode::List but append ";" to each line */
HTML, /* Generate an XHTML table */
INSERT, /* Generate SQL "insert" statements */
QUOTE, /* Quote values as for SQL */
TCL, /* Generate ANSI-C or TCL quoted elements */
CSV, /* Quote strings, numbers are plain */
EXPLAIN, /* Like RenderMode::Column, but do not truncate data */
ASCII, /* Use ASCII unit and record separators (0x1F/0x1E) */
PRETTY, /* Pretty-print schemas */
EQP, /* Converts EXPLAIN QUERY PLAN output into a graph */
JSON, /* Output JSON */
MARKDOWN, /* Markdown formatting */
TABLE, /* MySQL-style table formatting */
BOX, /* Unicode box-drawing characters */
LATEX, /* Latex tabular formatting */
TRASH, /* Discard output */
JSONLINES, /* Output JSON Lines */
DUCKBOX /* Unicode box drawing - using DuckDB's own renderer */
};
enum class PrintOutput { STDOUT, STDERR };
enum class InputMode { STANDARD, FILE };
enum class LargeNumberRendering { NONE = 0, FOOTER = 1, ALL = 2, DEFAULT = 3 };
/*
** These are the allowed shellFlgs values
*/
#define SHFLG_Pagecache 0x00000001 /* The --pagecache option is used */
#define SHFLG_Lookaside 0x00000002 /* Lookaside memory is used */
#define SHFLG_Backslash 0x00000004 /* The --backslash option is used */
#define SHFLG_PreserveRowid 0x00000008 /* .dump preserves rowid values */
#define SHFLG_Newlines 0x00000010 /* .dump --newline flag */
#define SHFLG_CountChanges 0x00000020 /* .changes setting */
#define SHFLG_Echo 0x00000040 /* .echo or --echo setting */
#define SHFLG_HeaderSet 0x00000080 /* .header has been used */
/* ctype macros that work with signed characters */
#define IsSpace(X) duckdb::StringUtil::CharacterIsSpace((unsigned char)X)
#define IsDigit(X) isdigit((unsigned char)X)
#define ToLower(X) (char)tolower((unsigned char)X)
/*
** State information about the database connection is contained in an
** instance of the following structure.
*/
struct ShellState {
public:
ShellState();
sqlite3 *db = nullptr; /* The database */
uint8_t openMode = 0; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
uint8_t doXdgOpen = 0; /* Invoke start/open/xdg-open in output_reset() */
int outCount = 0; /* Revert to stdout when reaching zero */
int lineno = 0; /* Line number of last line read from in */
int openFlags = 0; /* Additional flags to open. (SQLITE_OPEN_NOFOLLOW) */
FILE *in = nullptr; /* Read commands from this stream */
FILE *out = nullptr; /* Write results here */
int nErr = 0; /* Number of errors seen */
RenderMode mode = RenderMode::LINE; /* An output mode setting */
RenderMode modePrior = RenderMode::LINE; /* Saved mode */
RenderMode cMode = RenderMode::LINE; /* temporary output mode for the current query */
RenderMode normalMode = RenderMode::LINE; /* Output mode before ".explain on" */
bool showHeader = false; /* True to show column names in List or Column mode */
unsigned shellFlgs = 0; /* Various flags */
unsigned priorShFlgs = 0; /* Saved copy of flags */
int64_t szMax = 0; /* --maxsize argument to .open */
char *zDestTable = nullptr; /* Name of destination table when RenderMode::Insert */
char *zTempFile = nullptr; /* Temporary file that might need deleting */
string colSeparator; /* Column separator character for several modes */
string rowSeparator; /* Row separator character for RenderMode::Ascii */
string colSepPrior; /* Saved column separator */
string rowSepPrior; /* Saved row separator */
vector<int> colWidth; /* Requested width of each column in columnar modes */
string nullValue; /* The text to print when a NULL comes back from the database */
int columns = 0; /* Column-wise DuckBox rendering */
string outfile; /* Filename for *out */
string zDbFilename; /* name of the database file */
sqlite3_stmt *pStmt = nullptr; /* Current statement if any. */
FILE *pLog = nullptr; /* Write log output here */
size_t max_rows = 0; /* The maximum number of rows to render in DuckBox mode */
size_t max_width = 0; /* The maximum number of characters to render horizontally in DuckBox mode */
//! Decimal separator (if any)
char decimal_separator = '\0';
//! Thousand separator (if any)
char thousand_separator = '\0';
//! When to use formatting of large numbers (in DuckBox mode)
LargeNumberRendering large_number_rendering = LargeNumberRendering::DEFAULT;
//! The command to execute when `-ui` is passed in
string ui_command = "CALL start_ui()";
public:
void PushOutputMode();
void PopOutputMode();
void OutputCSV(const char *z, int bSep);
void PrintRowSeparator(idx_t nArg, const char *zSep, const vector<idx_t> &actualWidth);
void PrintMarkdownSeparator(idx_t nArg, const char *zSep, const vector<int> &colTypes,
const vector<idx_t> &actualWidth);
void OutputCString(const char *z);
void OutputQuotedString(const char *z);
void OutputQuotedEscapedString(const char *z);
void OutputHexBlob(const void *pBlob, int nBlob);
void PrintSchemaLine(const char *z, const char *zTail);
void PrintSchemaLineN(char *z, int n, const char *zTail);
void PrintOptionallyQuotedIdentifier(const char *z);
bool IsNumber(const char *z, int *realnum);
void OutputJSONString(const char *z, int n);
void PrintDashes(idx_t N);
void UTF8WidthPrint(FILE *pOut, idx_t w, const string &str, bool right_align);
bool SetOutputMode(const char *mode, const char *tbl_name);
bool ImportData(const char **azArg, idx_t nArg);
bool OpenDatabase(const char **azArg, idx_t nArg);
bool SetOutputFile(const char **azArg, idx_t nArg, char output_mode);
bool ReadFromFile(const string &file);
bool DisplaySchemas(const char **azArg, idx_t nArg);
MetadataResult DisplayEntries(const char **azArg, idx_t nArg, char type);
void ShowConfiguration();
idx_t RenderLength(const char *z);
idx_t RenderLength(const string &str);
void SetBinaryMode();
void SetTextMode();
static idx_t StringLength(const char *z);
void SetTableName(const char *zName);
int RunTableDumpQuery(const char *zSelect);
void PrintValue(const char *str);
void Print(PrintOutput output, const char *str);
void Print(PrintOutput output, const string &str);
void Print(const char *str);
void Print(const string &str);
void PrintPadded(const char *str, idx_t len);
bool ColumnTypeIsInteger(const char *type);
string strdup_handle_newline(const char *z);
ColumnarResult ExecuteColumnar(sqlite3_stmt *pStmt);
unique_ptr<ColumnRenderer> GetColumnRenderer();
unique_ptr<RowRenderer> GetRowRenderer();
unique_ptr<RowRenderer> GetRowRenderer(RenderMode mode);
void ExecutePreparedStatementColumnar(sqlite3_stmt *pStmt);
char **TableColumnList(const char *zTab);
void ExecutePreparedStatement(sqlite3_stmt *pStmt);
void PrintDatabaseError(const char *zErr);
int ShellDatabaseError(sqlite3 *db);
int RunInitialCommand(char *sql, bool bail);
int RenderRow(RowRenderer &renderer, RowResult &result);
int ExecuteSQL(const char *zSql, /* SQL to be evaluated */
char **pzErrMsg /* Error msg written here */
);
int RunSchemaDumpQuery(const char *zQuery);
void OpenDB(int openFlags);
void SetOrClearFlag(unsigned mFlag, const char *zArg);
bool ShellHasFlag(int flag) {
return (shellFlgs & flag) != 0;
}
void ShellSetFlag(int flag) {
shellFlgs |= flag;
}
void ShellClearFlag(int flag) {
shellFlgs &= ~flag;
}
void ResetOutput();
void ClearTempFile();
void NewTempFile(const char *zSuffix);
int DoMetaCommand(char *zLine);
int RunOneSqlLine(InputMode mode, char *zSql);
string GetDefaultDuckDBRC();
bool ProcessDuckDBRC(const char *file);
bool ProcessFile(const string &file, bool is_duckdb_rc = false);
int ProcessInput(InputMode mode);
};
} // namespace duckdb_shell

7
external/duckdb/tools/shell/linenoise/CMakeLists.txt vendored Normal file
View File

@@ -0,0 +1,7 @@
include_directories(include)
add_library(duckdb_linenoise OBJECT highlighting.cpp history.cpp linenoise.cpp
linenoise-c.cpp rendering.cpp terminal.cpp)
set(SHELL_SOURCES
${SHELL_SOURCES} $<TARGET_OBJECTS:duckdb_linenoise>
PARENT_SCOPE)

25
external/duckdb/tools/shell/linenoise/LICENSE vendored Normal file
View File

@@ -0,0 +1,25 @@
Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com>
Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

270
external/duckdb/tools/shell/linenoise/highlighting.cpp vendored Normal file
View File

@@ -0,0 +1,270 @@
#include "linenoise.hpp"
#include "linenoise.h"
#include "highlighting.hpp"
#include "duckdb/parser/parser.hpp"
#include "duckdb/common/string.hpp"
#if defined(_WIN32) || defined(__WIN32__) || defined(WIN32)
// disable highlighting on windows (for now?)
#define DISABLE_HIGHLIGHT
#endif
namespace duckdb {
#ifdef DISABLE_HIGHLIGHT
static int enableHighlighting = 0;
#else
static int enableHighlighting = 1;
#endif
struct Color {
const char *color_name;
const char *highlight;
};
static Color terminal_colors[] = {{"red", "\033[31m"}, {"green", "\033[32m"},
{"yellow", "\033[33m"}, {"blue", "\033[34m"},
{"magenta", "\033[35m"}, {"cyan", "\033[36m"},
{"white", "\033[37m"}, {"brightblack", "\033[90m"},
{"brightred", "\033[91m"}, {"brightgreen", "\033[92m"},
{"brightyellow", "\033[93m"}, {"brightblue", "\033[94m"},
{"brightmagenta", "\033[95m"}, {"brightcyan", "\033[96m"},
{"brightwhite", "\033[97m"}, {nullptr, nullptr}};
static std::string bold = "\033[1m";
static std::string underline = "\033[4m";
static std::string keyword = "\033[32m";
static std::string continuation_selected = "\033[32m";
static std::string constant = "\033[33m";
static std::string continuation = "\033[90m";
static std::string comment = "\033[90m";
static std::string error = "\033[31m";
static std::string reset = "\033[00m";
void Highlighting::Enable() {
enableHighlighting = 1;
}
void Highlighting::Disable() {
enableHighlighting = 0;
}
bool Highlighting::IsEnabled() {
return enableHighlighting;
}
const char *Highlighting::GetColorOption(const char *option) {
size_t index = 0;
while (terminal_colors[index].color_name) {
if (strcmp(terminal_colors[index].color_name, option) == 0) {
return terminal_colors[index].highlight;
}
index++;
}
return nullptr;
}
void Highlighting::SetHighlightingColor(HighlightingType type, const char *color) {
switch (type) {
case HighlightingType::KEYWORD:
keyword = color;
break;
case HighlightingType::CONSTANT:
constant = color;
break;
case HighlightingType::COMMENT:
comment = color;
break;
case HighlightingType::ERROR:
error = color;
break;
case HighlightingType::CONTINUATION:
continuation = color;
break;
case HighlightingType::CONTINUATION_SELECTED:
continuation_selected = color;
break;
}
}
static tokenType convertToken(duckdb::SimplifiedTokenType token_type) {
switch (token_type) {
case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_IDENTIFIER:
return tokenType::TOKEN_IDENTIFIER;
case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_NUMERIC_CONSTANT:
return tokenType::TOKEN_NUMERIC_CONSTANT;
case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_STRING_CONSTANT:
return tokenType::TOKEN_STRING_CONSTANT;
case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_OPERATOR:
return tokenType::TOKEN_OPERATOR;
case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_KEYWORD:
return tokenType::TOKEN_KEYWORD;
case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_COMMENT:
return tokenType::TOKEN_COMMENT;
default:
throw duckdb::InternalException("Unrecognized token type");
}
}
static vector<highlightToken> GetParseTokens(char *buf, size_t len) {
string sql(buf, len);
auto parseTokens = duckdb::Parser::Tokenize(sql);
vector<highlightToken> tokens;
for (auto &token : parseTokens) {
highlightToken new_token;
new_token.type = convertToken(token.type);
new_token.start = token.start;
tokens.push_back(new_token);
}
if (!tokens.empty() && tokens[0].start > 0) {
highlightToken new_token;
new_token.type = tokenType::TOKEN_IDENTIFIER;
new_token.start = 0;
tokens.insert(tokens.begin(), new_token);
}
if (tokens.empty() && sql.size() > 0) {
highlightToken new_token;
new_token.type = tokenType::TOKEN_IDENTIFIER;
new_token.start = 0;
tokens.push_back(new_token);
}
return tokens;
}
static vector<highlightToken> GetDotCommandTokens(char *buf, size_t len) {
vector<highlightToken> tokens;
// identifier token for the dot command itself
highlightToken dot_token;
dot_token.type = tokenType::TOKEN_KEYWORD;
dot_token.start = 0;
tokens.push_back(dot_token);
for (idx_t i = 0; i + 1 < len; i++) {
if (Linenoise::IsSpace(buf[i])) {
highlightToken argument_token;
argument_token.type = tokenType::TOKEN_STRING_CONSTANT;
argument_token.start = i + 1;
tokens.push_back(argument_token);
}
}
return tokens;
}
vector<highlightToken> Highlighting::Tokenize(char *buf, size_t len, bool is_dot_command, searchMatch *match) {
vector<highlightToken> tokens;
if (!is_dot_command) {
// SQL query - use parser to obtain tokens
tokens = GetParseTokens(buf, len);
} else {
// . command
tokens = GetDotCommandTokens(buf, len);
}
if (match) {
// we have a search match - insert it into the token list
// we want to insert a search token with start = match_start, end = match_end
// first figure out which token type we would have at match_end (if any)
for (size_t i = 0; i + 1 < tokens.size(); i++) {
if (tokens[i].start <= match->match_start && tokens[i + 1].start >= match->match_start) {
// this token begins after the search position, insert the token here
size_t token_position = i + 1;
auto end_type = tokens[i].type;
if (tokens[i].start == match->match_start) {
// exact start: only set the search match
tokens[i].search_match = true;
} else {
// non-exact start: add a new token
highlightToken search_token;
search_token.type = tokens[i].type;
search_token.start = match->match_start;
search_token.search_match = true;
tokens.insert(tokens.begin() + token_position, search_token);
token_position++;
}
// move forwards
while (token_position < tokens.size() && tokens[token_position].start < match->match_end) {
// this token is
// mark this token as a search token
end_type = tokens[token_position].type;
tokens[token_position].search_match = true;
token_position++;
}
if (token_position >= tokens.size() || tokens[token_position].start > match->match_end) {
// insert the token that marks the end of the search
highlightToken end_token;
end_token.type = end_type;
end_token.start = match->match_end;
tokens.insert(tokens.begin() + token_position, end_token);
token_position++;
}
break;
}
}
}
return tokens;
}
string Highlighting::HighlightText(char *buf, size_t len, size_t start_pos, size_t end_pos,
const vector<highlightToken> &tokens) {
duckdb::stringstream ss;
size_t prev_pos = 0;
for (size_t i = 0; i < tokens.size(); i++) {
size_t next = i + 1 < tokens.size() ? tokens[i + 1].start : len;
if (next < start_pos) {
// this token is not rendered at all
continue;
}
auto &token = tokens[i];
size_t start = token.start > start_pos ? token.start : start_pos;
size_t end = next > end_pos ? end_pos : next;
if (end <= start) {
continue;
}
if (prev_pos > start) {
#ifdef DEBUG
throw InternalException("ERROR - Rendering at position %llu after rendering at position %llu\n", start,
prev_pos);
#endif
Linenoise::Log("ERROR - Rendering at position %llu after rendering at position %llu\n", start, prev_pos);
continue;
}
prev_pos = start;
std::string text = std::string(buf + start, end - start);
if (token.search_match) {
ss << underline;
}
switch (token.type) {
case tokenType::TOKEN_KEYWORD:
ss << keyword << text << reset;
break;
case tokenType::TOKEN_NUMERIC_CONSTANT:
case tokenType::TOKEN_STRING_CONSTANT:
ss << constant << text << reset;
break;
case tokenType::TOKEN_CONTINUATION:
ss << continuation << text << reset;
break;
case tokenType::TOKEN_CONTINUATION_SELECTED:
ss << continuation_selected << text << reset;
break;
case tokenType::TOKEN_BRACKET:
ss << underline << text << reset;
break;
case tokenType::TOKEN_ERROR:
ss << error << text << reset;
break;
case tokenType::TOKEN_COMMENT:
ss << comment << text << reset;
break;
default:
ss << text;
if (token.search_match) {
ss << reset;
}
}
}
return ss.str();
}
} // namespace duckdb

367
external/duckdb/tools/shell/linenoise/history.cpp vendored Normal file
View File

@@ -0,0 +1,367 @@
#include "history.hpp"
#include "linenoise.hpp"
#include "terminal.hpp"
#include "duckdb_shell_wrapper.h"
#include "sqlite3.h"
#include "utf8proc_wrapper.hpp"
#include <sys/stat.h>
namespace duckdb {
#define LINENOISE_DEFAULT_HISTORY_MAX_LEN 1000
static idx_t history_max_len = LINENOISE_DEFAULT_HISTORY_MAX_LEN;
static idx_t history_len = 0;
static char **history = nullptr;
static char *history_file = nullptr;
/* Free the history, but does not reset it. Only used when we have to
* exit() to avoid memory leaks are reported by valgrind & co. */
void History::Free() {
if (history) {
for (idx_t j = 0; j < history_len; j++) {
free(history[j]);
}
free(history);
}
}
idx_t History::GetLength() {
return history_len;
}
const char *History::GetEntry(idx_t index) {
if (!history || index >= history_len) {
// FIXME: print debug message
return "";
}
return history[index];
}
void History::Overwrite(idx_t index, const char *new_entry) {
if (!history || index >= history_len) {
// FIXME: print debug message
return;
}
free(history[index]);
history[index] = strdup(new_entry);
}
void History::RemoveLastEntry() {
history_len--;
free(history[history_len]);
}
int History::Add(const char *line) {
return History::Add(line, strlen(line));
}
int History::Add(const char *line, idx_t len) {
char *linecopy;
if (history_max_len == 0) {
return 0;
}
/* Initialization on first call. */
if (history == nullptr) {
history = (char **)malloc(sizeof(char *) * history_max_len);
if (history == nullptr) {
return 0;
}
memset(history, 0, (sizeof(char *) * history_max_len));
}
/* Don't add duplicated lines. */
if (history_len && !strcmp(history[history_len - 1], line)) {
return 0;
}
if (!Utf8Proc::IsValid(line, len)) {
// don't add invalid UTF8 to history
return 0;
}
/* Add an heap allocated copy of the line in the history.
* If we reached the max length, remove the older line. */
if (!Terminal::IsMultiline()) {
// replace all newlines with spaces
linecopy = strdup(line);
if (!linecopy) {
return 0;
}
for (auto ptr = linecopy; *ptr; ptr++) {
if (*ptr == '\n' || *ptr == '\r') {
*ptr = ' ';
}
}
} else {
// replace all '\n' with '\r\n'
idx_t replaced_newline_count = 0;
idx_t len;
for (len = 0; line[len]; len++) {
if (line[len] == '\r' && line[len + 1] == '\n') {
// \r\n - skip past the \n
len++;
} else if (line[len] == '\n') {
replaced_newline_count++;
}
}
linecopy = (char *)malloc((len + replaced_newline_count + 1) * sizeof(char));
idx_t pos = 0;
for (len = 0; line[len]; len++) {
if (line[len] == '\r' && line[len + 1] == '\n') {
// \r\n - skip past the \n
linecopy[pos++] = '\r';
len++;
} else if (line[len] == '\n') {
linecopy[pos++] = '\r';
}
linecopy[pos++] = line[len];
}
linecopy[pos] = '\0';
}
if (history_len == history_max_len) {
free(history[0]);
memmove(history, history + 1, sizeof(char *) * (history_max_len - 1));
history_len--;
}
history[history_len] = linecopy;
history_len++;
if (history_file && strlen(line) > 0) {
// if there is a history file that we loaded from
// append to the history
// this way we can recover history in case of a crash
FILE *fp;
fp = fopen(history_file, "a");
if (fp == nullptr) {
return 1;
}
fprintf(fp, "%s\n", line);
fclose(fp);
}
return 1;
}
int History::SetMaxLength(idx_t len) {
char **new_entry;
if (len < 1) {
return 0;
}
if (history) {
idx_t tocopy = history_len;
new_entry = (char **)malloc(sizeof(char *) * len);
if (new_entry == nullptr) {
return 0;
}
/* If we can't copy everything, free the elements we'll not use. */
if (len < tocopy) {
for (idx_t j = 0; j < tocopy - len; j++) {
free(history[j]);
}
tocopy = len;
}
memset(new_entry, 0, sizeof(char *) * len);
memcpy(new_entry, history + (history_len - tocopy), sizeof(char *) * tocopy);
free(history);
history = new_entry;
}
history_max_len = len;
if (history_len > history_max_len) {
history_len = history_max_len;
}
return 1;
}
int History::Save(const char *filename) {
mode_t old_umask = umask(S_IXUSR | S_IRWXG | S_IRWXO);
FILE *fp;
fp = fopen(filename, "w");
umask(old_umask);
if (fp == nullptr) {
return -1;
}
chmod(filename, S_IRUSR | S_IWUSR);
for (idx_t j = 0; j < history_len; j++) {
fprintf(fp, "%s\n", history[j]);
}
fclose(fp);
return 0;
}
struct LineReader {
static constexpr idx_t LINE_BUFFER_SIZE = LINENOISE_MAX_LINE * 2ULL;
public:
LineReader() : fp(nullptr), filename(nullptr), end_of_file(false), position(0), capacity(0), total_read(0) {
line_buffer[LINENOISE_MAX_LINE] = '\0';
data_buffer[LINE_BUFFER_SIZE] = '\0';
}
bool Init(const char *filename_p) {
filename = filename_p;
fp = fopen(filename, "r");
return fp;
}
void Close() {
if (fp) {
fclose(fp);
fp = nullptr;
}
}
const char *GetLine() {
return line_buffer;
}
idx_t GetNextNewline() {
for (idx_t i = position; i < capacity; i++) {
if (data_buffer[i] == '\r' || data_buffer[i] == '\n') {
return i;
}
}
return capacity;
}
void SkipNewline() {
if (position >= capacity) {
// we are already at the end - fill the buffer
FillBuffer();
}
if (position < capacity && data_buffer[position] == '\n') {
position++;
}
}
bool NextLine() {
idx_t line_size = 0;
while (true) {
// find the next newline in the current buffer (if any)
idx_t i = GetNextNewline();
// copy over the data and move to the next byte
idx_t read_count = i - position;
if (line_size + read_count > LINENOISE_MAX_LINE) {
// exceeded max line size
// move on to next line and don't add to history
// skip to next newline
bool found_next_newline = false;
while (!found_next_newline && capacity > 0) {
i = GetNextNewline();
if (i < capacity) {
found_next_newline = true;
}
if (!found_next_newline) {
// read more data
FillBuffer();
}
}
if (!found_next_newline) {
// no newline found - skip
return false;
}
// newline found - adjust position and read next line
position = i + 1;
if (data_buffer[i] == '\r') {
// \r\n - skip the next byte as well
SkipNewline();
}
continue;
}
memcpy(line_buffer + line_size, data_buffer + position, read_count);
line_size += read_count;
if (i < capacity) {
// we're still within the buffer - this means we found a newline in the buffer
line_buffer[line_size] = '\0';
position = i + 1;
if (data_buffer[i] == '\r') {
// \r\n - skip the next byte as well
SkipNewline();
}
if (line_size == 0 || !Utf8Proc::IsValid(line_buffer, line_size)) {
// line is empty OR not valid UTF8
// move on to next line and don't add to history
line_size = 0;
continue;
}
return true;
}
// we need to read more data - fill up the buffer
FillBuffer();
if (capacity == 0) {
// no more data available - return true if there is anything we copied over (i.e. part of the next line)
return line_size > 0;
}
}
}
void FillBuffer() {
if (end_of_file || !fp) {
return;
}
size_t read_data = fread(data_buffer, 1, LINE_BUFFER_SIZE, fp);
position = 0;
capacity = read_data;
total_read += read_data;
data_buffer[read_data] = '\0';
if (read_data == 0) {
end_of_file = true;
}
if (total_read >= LINENOISE_MAX_HISTORY) {
fprintf(stderr, "History file \"%s\" exceeds maximum history file size of %d MB - skipping full load\n",
filename, LINENOISE_MAX_HISTORY / 1024 / 1024);
capacity = 0;
end_of_file = true;
}
}
private:
FILE *fp;
const char *filename;
char line_buffer[LINENOISE_MAX_LINE + 1];
char data_buffer[LINE_BUFFER_SIZE + 1];
bool end_of_file;
idx_t position;
idx_t capacity;
idx_t total_read;
};
int History::Load(const char *filename) {
LineReader reader;
if (!reader.Init(filename)) {
return -1;
}
std::string result;
while (reader.NextLine()) {
auto buf = reader.GetLine();
if (result.empty() && buf[0] == '.') {
// if the first character is a dot this is a dot command
// add the full line to the history
History::Add(buf);
continue;
}
// else we are parsing a SQL statement
result += buf;
if (sqlite3_complete(result.c_str())) {
// this line contains a full SQL statement - add it to the history
History::Add(result.c_str(), result.size());
result = std::string();
continue;
}
// the result does not contain a full SQL statement - add a newline deliminator and move on to the next line
result += "\r\n";
}
reader.Close();
history_file = strdup(filename);
return 0;
}
} // namespace duckdb

50
external/duckdb/tools/shell/linenoise/include/highlighting.hpp vendored Normal file
View File

@@ -0,0 +1,50 @@
//===----------------------------------------------------------------------===//
// DuckDB
//
// highlighting.hpp
//
//
//===----------------------------------------------------------------------===//
#pragma once
#include "duckdb/common/common.hpp"
namespace duckdb {
struct searchMatch;
enum class tokenType : uint8_t {
TOKEN_IDENTIFIER,
TOKEN_NUMERIC_CONSTANT,
TOKEN_STRING_CONSTANT,
TOKEN_OPERATOR,
TOKEN_KEYWORD,
TOKEN_COMMENT,
TOKEN_CONTINUATION,
TOKEN_CONTINUATION_SELECTED,
TOKEN_BRACKET,
TOKEN_ERROR
};
enum class HighlightingType { KEYWORD, CONSTANT, COMMENT, ERROR, CONTINUATION, CONTINUATION_SELECTED };
struct highlightToken {
tokenType type;
size_t start = 0;
bool search_match = false;
};
class Highlighting {
public:
static void Enable();
static void Disable();
static bool IsEnabled();
static const char *GetColorOption(const char *option);
static void SetHighlightingColor(HighlightingType type, const char *color);
static vector<highlightToken> Tokenize(char *buf, size_t len, bool is_dot_command, searchMatch *match);
static string HighlightText(char *buf, size_t len, size_t start_pos, size_t end_pos,
const vector<highlightToken> &tokens);
};
} // namespace duckdb

29
external/duckdb/tools/shell/linenoise/include/history.hpp vendored Normal file
View File

@@ -0,0 +1,29 @@
//===----------------------------------------------------------------------===//
// DuckDB
//
// history.hpp
//
//
//===----------------------------------------------------------------------===//
#pragma once
#include "duckdb/common/common.hpp"
namespace duckdb {
class History {
public:
static void Free();
static idx_t GetLength();
static const char *GetEntry(idx_t index);
static void Overwrite(idx_t index, const char *new_entry);
static void RemoveLastEntry();
static int Add(const char *line);
static int Add(const char *line, idx_t len);
static int SetMaxLength(idx_t len);
static int Save(const char *filename);
static int Load(const char *filename);
};
} // namespace duckdb

76
external/duckdb/tools/shell/linenoise/include/linenoise.h vendored Normal file
View File

@@ -0,0 +1,76 @@
/* linenoise.h -- VERSION 1.0
*
* Guerrilla line editing library against the idea that a line editing lib
* needs to be 20,000 lines of C code.
*
* See linenoise.c for more information.
*
* ------------------------------------------------------------------------
*
* Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com>
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __LINENOISE_H
#define __LINENOISE_H
#ifdef __cplusplus
extern "C" {
#endif
typedef struct linenoiseCompletions {
size_t len;
char **cvec;
} linenoiseCompletions;
typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *);
typedef char *(linenoiseHintsCallback)(const char *, int *color, int *bold);
typedef void(linenoiseFreeHintsCallback)(void *);
void linenoiseSetCompletionCallback(linenoiseCompletionCallback *);
void linenoiseSetHintsCallback(linenoiseHintsCallback *);
void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *);
void linenoiseAddCompletion(linenoiseCompletions *, const char *);
char *linenoise(const char *prompt);
void linenoiseFree(void *ptr);
int linenoiseParseOption(const char **azArg, int nArg, const char **out_error);
int linenoiseHistoryAdd(const char *line);
int linenoiseHistorySetMaxLen(int len);
int linenoiseHistorySave(const char *filename);
int linenoiseHistoryLoad(const char *filename);
void linenoiseClearScreen(void);
void linenoiseSetMultiLine(int ml);
size_t linenoiseComputeRenderWidth(const char *buf, size_t len);
int linenoiseGetRenderPosition(const char *buf, size_t len, int max_width, int *n);
void linenoiseSetPrompt(const char *continuation, const char *continuationSelected);
#ifdef __cplusplus
}
#endif
#endif /* __LINENOISE_H */

190
external/duckdb/tools/shell/linenoise/include/linenoise.hpp vendored Normal file
View File

@@ -0,0 +1,190 @@
//===----------------------------------------------------------------------===//
// DuckDB
//
// linenoise.hpp
//
//
//===----------------------------------------------------------------------===//
#pragma once
#include "duckdb/common/common.hpp"
#include "duckdb/common/exception.hpp"
#include "terminal.hpp"
#include "linenoise.h"
#define LINENOISE_MAX_LINE 204800
#define LINENOISE_MAX_HISTORY 104857600
#define LINENOISE_EDITOR
namespace duckdb {
struct highlightToken;
struct AppendBuffer;
enum class HistoryScrollDirection : uint8_t {
LINENOISE_HISTORY_NEXT,
LINENOISE_HISTORY_PREV,
LINENOISE_HISTORY_START,
LINENOISE_HISTORY_END
};
enum class Capitalization : uint8_t { CAPITALIZE, LOWERCASE, UPPERCASE };
struct searchMatch {
size_t history_index;
size_t match_start;
size_t match_end;
};
struct Completion {
string completion;
idx_t cursor_pos;
};
struct TabCompletion {
vector<Completion> completions;
};
class Linenoise {
public:
Linenoise(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt);
public:
int Edit();
static void SetCompletionCallback(linenoiseCompletionCallback *fn);
static void SetHintsCallback(linenoiseHintsCallback *fn);
static void SetFreeHintsCallback(linenoiseFreeHintsCallback *fn);
static linenoiseHintsCallback *HintsCallback();
static linenoiseFreeHintsCallback *FreeHintsCallback();
static void SetPrompt(const char *continuation, const char *continuationSelected);
static size_t ComputeRenderWidth(const char *buf, size_t len);
static int GetRenderPosition(const char *buf, size_t len, int max_width, int *n);
static int ParseOption(const char **azArg, int nArg, const char **out_error);
int GetPromptWidth() const;
void RefreshLine();
int CompleteLine(EscapeSequence &current_sequence);
void InsertCharacter(char c);
int EditInsert(char c);
int EditInsertMulti(const char *c);
void EditMoveLeft();
void EditMoveRight();
void EditMoveWordLeft();
void EditMoveWordRight();
bool EditMoveRowUp();
bool EditMoveRowDown();
void EditMoveHome();
void EditMoveEnd();
void EditMoveStartOfLine();
void EditMoveEndOfLine();
void EditHistoryNext(HistoryScrollDirection dir);
void EditHistorySetIndex(idx_t index);
void EditDelete();
void EditBackspace();
void EditDeletePrevWord();
void EditDeleteNextWord();
void EditDeleteAll();
void EditCapitalizeNextWord(Capitalization capitalization);
void EditRemoveSpaces();
void EditSwapCharacter();
void EditSwapWord();
void StartSearch();
void CancelSearch();
char AcceptSearch(char nextCommand);
void PerformSearch();
void SearchPrev();
void SearchNext();
#ifdef LINENOISE_EDITOR
bool EditBufferWithEditor(const char *editor);
bool EditFileWithEditor(const string &file_name, const char *editor);
#endif
char Search(char c);
void RefreshMultiLine();
void RefreshSingleLine() const;
void RefreshSearch();
void RefreshShowHints(AppendBuffer &append_buffer, int plen) const;
size_t PrevChar() const;
size_t NextChar() const;
void NextPosition(const char *buf, size_t len, size_t &cpos, int &rows, int &cols, int plen) const;
void PositionToColAndRow(size_t target_pos, int &out_row, int &out_col, int &rows, int &cols) const;
size_t ColAndRowToPosition(int target_row, int target_col) const;
string AddContinuationMarkers(const char *buf, size_t len, int plen, int cursor_row,
vector<highlightToken> &tokens) const;
void AddErrorHighlighting(idx_t render_start, idx_t render_end, vector<highlightToken> &tokens) const;
bool AddCompletionMarker(const char *buf, idx_t len, string &result_buffer, vector<highlightToken> &tokens) const;
static bool IsNewline(char c);
static bool IsWordBoundary(char c);
static bool AllWhitespace(const char *z);
static bool IsSpace(char c);
TabCompletion TabComplete() const;
static void EnableCompletionRendering();
static void DisableCompletionRendering();
static void EnableErrorRendering();
static void DisableErrorRendering();
public:
static void LogTokens(const vector<highlightToken> &tokens);
#ifdef LINENOISE_LOGGING
// Logging
template <typename... Args>
static void Log(const string &msg, Args... params) {
std::vector<ExceptionFormatValue> values;
LogMessageRecursive(msg, values, params...);
}
static void LogMessageRecursive(const string &msg, std::vector<ExceptionFormatValue> &values);
template <class T, typename... Args>
static void LogMessageRecursive(const string &msg, std::vector<ExceptionFormatValue> &values, T param,
Args... params) {
values.push_back(ExceptionFormatValue::CreateFormatValue<T>(param));
LogMessageRecursive(msg, values, params...);
}
#else
template <typename... Args>
static void Log(const string &msg, Args... params) {
// nop
}
#endif
public:
int ifd; /* Terminal stdin file descriptor. */
int ofd; /* Terminal stdout file descriptor. */
char *buf; /* Edited line buffer. */
size_t buflen; /* Edited line buffer size. */
const char *prompt; /* Prompt to display. */
size_t plen; /* Prompt length. */
size_t pos; /* Current cursor position. */
size_t old_cursor_rows; /* Previous refresh cursor position. */
size_t len; /* Current edited line length. */
size_t y_scroll; /* The y scroll position (multiline mode) */
TerminalSize ws; /* Terminal size */
size_t maxrows; /* Maximum num of rows used so far (multiline mode) */
idx_t history_index; /* The history index we are currently editing. */
bool clear_screen; /* Whether we are clearing the screen */
bool continuation_markers; /* Whether or not to render continuation markers */
bool search; /* Whether or not we are searching our history */
bool render; /* Whether or not to re-render */
bool has_more_data; /* Whether or not there is more data available in the buffer (copy+paste)*/
bool insert; /* Whether or not the last action was inserting a new character */
std::string search_buf; //! The search buffer
std::vector<searchMatch> search_matches; //! The set of search matches in our history
size_t search_index; //! The current match index
};
} // namespace duckdb

122
external/duckdb/tools/shell/linenoise/include/terminal.hpp vendored Normal file
View File

@@ -0,0 +1,122 @@
//===----------------------------------------------------------------------===//
// DuckDB
//
// terminal.hpp
//
//
//===----------------------------------------------------------------------===//
#pragma once
#include "duckdb/common/common.hpp"
namespace duckdb {
enum KEY_ACTION {
KEY_NULL = 0, /* NULL */
CTRL_A = 1, /* Ctrl+a */
CTRL_B = 2, /* Ctrl-b */
CTRL_C = 3, /* Ctrl-c */
CTRL_D = 4, /* Ctrl-d */
CTRL_E = 5, /* Ctrl-e */
CTRL_F = 6, /* Ctrl-f */
CTRL_G = 7, /* Ctrl-g */
CTRL_H = 8, /* Ctrl-h */
TAB = 9, /* Tab */
CTRL_J = 10, /* Ctrl+j*/
CTRL_K = 11, /* Ctrl+k */
CTRL_L = 12, /* Ctrl+l */
ENTER = 13, /* Enter */
CTRL_N = 14, /* Ctrl-n */
CTRL_O = 15, /* Ctrl-O */
CTRL_P = 16, /* Ctrl-p */
CTRL_R = 18, /* Ctrl-r */
CTRL_S = 19, /* Ctrl-s */
CTRL_T = 20, /* Ctrl-t */
CTRL_U = 21, /* Ctrl+u */
CTRL_W = 23, /* Ctrl+w */
CTRL_X = 24, /* Ctrl+x */
CTRL_Y = 25, /* Ctrl+y */
CTRL_Z = 26, /* Ctrl+z */
ESC = 27, /* Escape */
BACKSPACE = 127 /* Backspace */
};
enum class EscapeSequence {
INVALID = 0,
UNKNOWN = 1,
CTRL_MOVE_BACKWARDS,
CTRL_MOVE_FORWARDS,
HOME,
END,
UP,
DOWN,
RIGHT,
LEFT,
DELETE,
SHIFT_TAB,
ESCAPE,
ALT_A,
ALT_B,
ALT_C,
ALT_D,
ALT_E,
ALT_F,
ALT_G,
ALT_H,
ALT_I,
ALT_J,
ALT_K,
ALT_L,
ALT_M,
ALT_N,
ALT_O,
ALT_P,
ALT_Q,
ALT_R,
ALT_S,
ALT_T,
ALT_U,
ALT_V,
ALT_W,
ALT_X,
ALT_Y,
ALT_Z,
ALT_BACKSPACE,
ALT_LEFT_ARROW,
ALT_RIGHT_ARROW,
ALT_BACKSLASH,
};
struct TerminalSize {
int ws_col = 0;
int ws_row = 0;
};
class Terminal {
public:
static int IsUnsupportedTerm();
static int EnableRawMode();
static void DisableRawMode();
static bool IsMultiline();
static void SetMultiLine(int ml);
static void ClearScreen();
static void Beep();
static bool IsAtty();
static int HasMoreData(int fd);
static TerminalSize GetTerminalSize();
static char *EditNoTTY();
static int EditRaw(char *buf, size_t buflen, const char *prompt);
static EscapeSequence ReadEscapeSequence(int ifd);
private:
static TerminalSize TryMeasureTerminalSize();
static TerminalSize GetCursorPosition();
static idx_t ReadEscapeSequence(int ifd, char sequence[]);
};
} // namespace duckdb

154
external/duckdb/tools/shell/linenoise/linenoise-c.cpp vendored Normal file
View File

@@ -0,0 +1,154 @@
#include "linenoise.hpp"
#include "linenoise.h"
#include "history.hpp"
#include "terminal.hpp"
using duckdb::History;
using duckdb::idx_t;
using duckdb::Linenoise;
using duckdb::Terminal;
/* The high level function that is the main API of the linenoise library.
* This function checks if the terminal has basic capabilities, just checking
* for a blacklist of stupid terminals, and later either calls the line
* editing function or uses dummy fgets() so that you will be able to type
* something even in the most desperate of the conditions. */
char *linenoise(const char *prompt) {
char buf[LINENOISE_MAX_LINE];
int count;
if (!Terminal::IsAtty()) {
/* Not a tty: read from file / pipe. In this mode we don't want any
* limit to the line size, so we call a function to handle that. */
return Terminal::EditNoTTY();
} else if (Terminal::IsUnsupportedTerm()) {
size_t len;
printf("%s", prompt);
fflush(stdout);
if (fgets(buf, LINENOISE_MAX_LINE, stdin) == NULL) {
return NULL;
}
len = strlen(buf);
while (len && (buf[len - 1] == '\n' || buf[len - 1] == '\r')) {
len--;
buf[len] = '\0';
}
return strdup(buf);
} else {
count = Terminal::EditRaw(buf, LINENOISE_MAX_LINE, prompt);
if (count == -1) {
return NULL;
}
return strdup(buf);
}
}
/* This is just a wrapper the user may want to call in order to make sure
* the linenoise returned buffer is freed with the same allocator it was
* created with. Useful when the main program is using an alternative
* allocator. */
void linenoiseFree(void *ptr) {
free(ptr);
}
/* ================================ History ================================= */
/* This is the API call to add a new entry in the linenoise history.
* It uses a fixed array of char pointers that are shifted (memmoved)
* when the history max length is reached in order to remove the older
* entry and make room for the new one, so it is not exactly suitable for huge
* histories, but will work well for a few hundred of entries.
*
* Using a circular buffer is smarter, but a bit more complex to handle. */
int linenoiseHistoryAdd(const char *line) {
return History::Add(line);
}
/* Set the maximum length for the history. This function can be called even
* if there is already some history, the function will make sure to retain
* just the latest 'len' elements if the new history length value is smaller
* than the amount of items already inside the history. */
int linenoiseHistorySetMaxLen(int len) {
if (len < 0) {
return 0;
}
return History::SetMaxLength(idx_t(len));
}
/* Save the history in the specified file. On success 0 is returned
* otherwise -1 is returned. */
int linenoiseHistorySave(const char *filename) {
return History::Save(filename);
}
/* Load the history from the specified file. If the file does not exist
* zero is returned and no operation is performed.
*
* If the file exists and the operation succeeded 0 is returned, otherwise
* on error -1 is returned. */
int linenoiseHistoryLoad(const char *filename) {
return History::Load(filename);
}
/* Register a callback function to be called for tab-completion. */
void linenoiseSetCompletionCallback(linenoiseCompletionCallback *fn) {
Linenoise::SetCompletionCallback(fn);
}
/* Register a hits function to be called to show hits to the user at the
* right of the prompt. */
void linenoiseSetHintsCallback(linenoiseHintsCallback *fn) {
Linenoise::SetHintsCallback(fn);
}
/* Register a function to free the hints returned by the hints callback
* registered with linenoiseSetHintsCallback(). */
void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *fn) {
Linenoise::SetFreeHintsCallback(fn);
}
void linenoiseSetMultiLine(int ml) {
Terminal::SetMultiLine(ml);
}
void linenoiseSetPrompt(const char *continuation, const char *continuationSelected) {
Linenoise::SetPrompt(continuation, continuationSelected);
}
/* This function is used by the callback function registered by the user
* in order to add completion options given the input string when the
* user typed <tab>. See the example.c source code for a very easy to
* understand example. */
void linenoiseAddCompletion(linenoiseCompletions *lc, const char *str) {
size_t len = strlen(str);
char *copy, **cvec;
copy = (char *)malloc(len + 1);
if (copy == NULL)
return;
memcpy(copy, str, len + 1);
cvec = (char **)realloc(lc->cvec, sizeof(char *) * (lc->len + 1));
if (cvec == NULL) {
free(copy);
return;
}
lc->cvec = cvec;
lc->cvec[lc->len++] = copy;
}
size_t linenoiseComputeRenderWidth(const char *buf, size_t len) {
return Linenoise::ComputeRenderWidth(buf, len);
}
int linenoiseGetRenderPosition(const char *buf, size_t len, int max_width, int *n) {
return Linenoise::GetRenderPosition(buf, len, max_width, n);
}
void linenoiseClearScreen(void) {
Terminal::ClearScreen();
}
int linenoiseParseOption(const char **azArg, int nArg, const char **out_error) {
return Linenoise::ParseOption(azArg, nArg, out_error);
}

1622
external/duckdb/tools/shell/linenoise/linenoise.cpp vendored Normal file
View File

File diff suppressed because it is too large Load Diff

975
external/duckdb/tools/shell/linenoise/rendering.cpp vendored Normal file
View File

@@ -0,0 +1,975 @@
#include "linenoise.hpp"
#include "highlighting.hpp"
#include "history.hpp"
#include "utf8proc_wrapper.hpp"
#include <unistd.h>
namespace duckdb {
static const char *continuationPrompt = "> ";
static const char *continuationSelectedPrompt = "> ";
static bool enableCompletionRendering = false;
static bool enableErrorRendering = true;
void Linenoise::EnableCompletionRendering() {
enableCompletionRendering = true;
}
void Linenoise::DisableCompletionRendering() {
enableCompletionRendering = false;
}
void Linenoise::EnableErrorRendering() {
enableErrorRendering = true;
}
void Linenoise::DisableErrorRendering() {
enableErrorRendering = false;
}
/* =========================== Line editing ================================= */
/* We define a very simple "append buffer" structure, that is an heap
* allocated string where we can append to. This is useful in order to
* write all the escape sequences in a buffer and flush them to the standard
* output in a single call, to avoid flickering effects. */
struct AppendBuffer {
void Append(const char *s, idx_t len) {
buffer.append(s, len);
}
void Append(const char *s) {
buffer.append(s);
}
void Write(int fd) {
if (write(fd, buffer.c_str(), buffer.size()) == -1) {
/* Can't recover from write error. */
Linenoise::Log("%s", "Failed to write buffer\n");
}
}
private:
std::string buffer;
};
void Linenoise::SetPrompt(const char *continuation, const char *continuationSelected) {
continuationPrompt = continuation;
continuationSelectedPrompt = continuationSelected;
}
/* Helper of refreshSingleLine() and refreshMultiLine() to show hints
* to the right of the prompt. */
void Linenoise::RefreshShowHints(AppendBuffer &append_buffer, int plen) const {
char seq[64];
auto hints_callback = Linenoise::HintsCallback();
if (hints_callback && plen + len < size_t(ws.ws_col)) {
int color = -1, bold = 0;
char *hint = hints_callback(buf, &color, &bold);
if (hint) {
int hintlen = strlen(hint);
int hintmaxlen = ws.ws_col - (plen + len);
if (hintlen > hintmaxlen) {
hintlen = hintmaxlen;
}
if (bold == 1 && color == -1)
color = 37;
if (color != -1 || bold != 0) {
snprintf(seq, 64, "\033[%d;%d;49m", bold, color);
} else {
seq[0] = '\0';
}
append_buffer.Append(seq, strlen(seq));
append_buffer.Append(hint, hintlen);
if (color != -1 || bold != 0) {
append_buffer.Append("\033[0m");
}
/* Call the function to free the hint returned. */
auto free_hints_callback = Linenoise::FreeHintsCallback();
if (free_hints_callback) {
free_hints_callback(hint);
}
}
}
}
static void renderText(size_t &render_pos, char *&buf, size_t &len, size_t pos, size_t cols, size_t plen,
std::string &highlight_buffer, bool highlight, searchMatch *match = nullptr) {
if (duckdb::Utf8Proc::IsValid(buf, len)) {
// utf8 in prompt, handle rendering
size_t remaining_render_width = cols - plen - 1;
size_t start_pos = 0;
size_t cpos = 0;
size_t prev_pos = 0;
size_t total_render_width = 0;
while (cpos < len) {
size_t char_render_width = duckdb::Utf8Proc::RenderWidth(buf, len, cpos);
prev_pos = cpos;
cpos = duckdb::Utf8Proc::NextGraphemeCluster(buf, len, cpos);
total_render_width += cpos - prev_pos;
if (total_render_width >= remaining_render_width) {
// character does not fit anymore! we need to figure something out
if (prev_pos >= pos) {
// we passed the cursor: break
cpos = prev_pos;
break;
} else {
// we did not pass the cursor yet! remove characters from the start until it fits again
while (total_render_width >= remaining_render_width) {
size_t start_char_width = duckdb::Utf8Proc::RenderWidth(buf, len, start_pos);
size_t new_start = duckdb::Utf8Proc::NextGraphemeCluster(buf, len, start_pos);
total_render_width -= new_start - start_pos;
start_pos = new_start;
render_pos -= start_char_width;
}
}
}
if (prev_pos < pos) {
render_pos += char_render_width;
}
}
if (highlight) {
bool is_dot_command = buf[0] == '.';
auto tokens = Highlighting::Tokenize(buf, len, is_dot_command, match);
highlight_buffer = Highlighting::HighlightText(buf, len, start_pos, cpos, tokens);
buf = (char *)highlight_buffer.c_str();
len = highlight_buffer.size();
} else {
buf = buf + start_pos;
len = cpos - start_pos;
}
} else {
// invalid UTF8: fallback
while ((plen + pos) >= cols) {
buf++;
len--;
pos--;
}
while (plen + len > cols) {
len--;
}
render_pos = pos;
}
}
/* Single line low level line refresh.
*
* Rewrite the currently edited line accordingly to the buffer content,
* cursor position, and number of columns of the terminal. */
void Linenoise::RefreshSingleLine() const {
char seq[64];
size_t plen = GetPromptWidth();
int fd = ofd;
char *render_buf = buf;
size_t render_len = len;
size_t render_pos = 0;
std::string highlight_buffer;
renderText(render_pos, render_buf, render_len, pos, ws.ws_col, plen, highlight_buffer, Highlighting::IsEnabled());
AppendBuffer append_buffer;
/* Cursor to left edge */
append_buffer.Append("\r");
/* Write the prompt and the current buffer content */
append_buffer.Append(prompt);
append_buffer.Append(render_buf, render_len);
/* Show hits if any. */
RefreshShowHints(append_buffer, plen);
/* Erase to right */
append_buffer.Append("\x1b[0K");
/* Move cursor to original position. */
snprintf(seq, 64, "\r\x1b[%dC", (int)(render_pos + plen));
append_buffer.Append(seq);
append_buffer.Write(fd);
}
void Linenoise::RefreshSearch() {
std::string search_prompt;
static const size_t SEARCH_PROMPT_RENDER_SIZE = 28;
std::string no_matches_text = "(no matches)";
bool no_matches = search_index >= search_matches.size();
if (search_buf.empty()) {
search_prompt = "search" + std::string(SEARCH_PROMPT_RENDER_SIZE - 8, ' ') + "> ";
no_matches_text = "(type to search)";
} else {
std::string search_text;
std::string matches_text;
search_text += search_buf;
if (!no_matches) {
matches_text += std::to_string(search_index + 1);
matches_text += "/" + std::to_string(search_matches.size());
}
size_t search_text_length = ComputeRenderWidth(search_text.c_str(), search_text.size());
size_t matches_text_length = ComputeRenderWidth(matches_text.c_str(), matches_text.size());
size_t total_text_length = search_text_length + matches_text_length;
if (total_text_length < SEARCH_PROMPT_RENDER_SIZE - 2) {
// search text is short: we can render the entire search text
search_prompt = search_text;
search_prompt += std::string(SEARCH_PROMPT_RENDER_SIZE - 2 - total_text_length, ' ');
search_prompt += matches_text;
search_prompt += "> ";
} else {
// search text length is too long to fit: truncate
bool render_matches = matches_text_length < SEARCH_PROMPT_RENDER_SIZE - 8;
char *search_buf = (char *)search_text.c_str();
size_t search_len = search_text.size();
size_t search_render_pos = 0;
size_t max_render_size = SEARCH_PROMPT_RENDER_SIZE - 3;
if (render_matches) {
max_render_size -= matches_text_length;
}
std::string highlight_buffer;
renderText(search_render_pos, search_buf, search_len, search_len, max_render_size, 0, highlight_buffer,
false);
search_prompt = std::string(search_buf, search_len);
for (size_t i = search_render_pos; i < max_render_size; i++) {
search_prompt += " ";
}
if (render_matches) {
search_prompt += matches_text;
}
search_prompt += "> ";
}
}
auto oldHighlighting = Highlighting::IsEnabled();
Linenoise clone = *this;
prompt = search_prompt.c_str();
plen = search_prompt.size();
if (no_matches || search_buf.empty()) {
// if there are no matches render the no_matches_text
buf = (char *)no_matches_text.c_str();
len = no_matches_text.size();
pos = 0;
// don't highlight the "no_matches" text
Highlighting::Disable();
} else {
// if there are matches render the current history item
auto search_match = search_matches[search_index];
auto history_index = search_match.history_index;
auto cursor_position = search_match.match_end;
buf = (char *)History::GetEntry(history_index);
len = strlen(buf);
pos = cursor_position;
}
RefreshLine();
if (oldHighlighting) {
Highlighting::Enable();
}
buf = clone.buf;
len = clone.len;
pos = clone.pos;
prompt = clone.prompt;
plen = clone.plen;
}
string Linenoise::AddContinuationMarkers(const char *buf, size_t len, int plen, int cursor_row,
vector<highlightToken> &tokens) const {
std::string result;
int rows = 1;
int cols = plen;
size_t cpos = 0;
size_t prev_pos = 0;
size_t extra_bytes = 0; // extra bytes introduced
size_t token_position = 0; // token position
vector<highlightToken> new_tokens;
new_tokens.reserve(tokens.size());
while (cpos < len) {
bool is_newline = IsNewline(buf[cpos]);
NextPosition(buf, len, cpos, rows, cols, plen);
for (; prev_pos < cpos; prev_pos++) {
result += buf[prev_pos];
}
if (is_newline) {
bool is_cursor_row = rows == cursor_row;
const char *prompt = is_cursor_row ? continuationSelectedPrompt : continuationPrompt;
if (!continuation_markers) {
prompt = "";
}
size_t continuationLen = strlen(prompt);
size_t continuationRender = ComputeRenderWidth(prompt, continuationLen);
// pad with spaces prior to prompt
for (int i = int(continuationRender); i < plen; i++) {
result += " ";
}
result += prompt;
size_t continuationBytes = plen - continuationRender + continuationLen;
if (token_position < tokens.size()) {
for (; token_position < tokens.size(); token_position++) {
if (tokens[token_position].start >= cpos) {
// not there yet
break;
}
tokens[token_position].start += extra_bytes;
new_tokens.push_back(tokens[token_position]);
}
tokenType prev_type = tokenType::TOKEN_IDENTIFIER;
if (token_position > 0 && token_position < tokens.size() + 1) {
prev_type = tokens[token_position - 1].type;
}
highlightToken token;
token.start = cpos + extra_bytes;
token.type = is_cursor_row ? tokenType::TOKEN_CONTINUATION_SELECTED : tokenType::TOKEN_CONTINUATION;
token.search_match = false;
new_tokens.push_back(token);
token.start = cpos + extra_bytes + continuationBytes;
token.type = prev_type;
token.search_match = false;
new_tokens.push_back(token);
}
extra_bytes += continuationBytes;
}
}
for (; prev_pos < cpos; prev_pos++) {
result += buf[prev_pos];
}
for (; token_position < tokens.size(); token_position++) {
tokens[token_position].start += extra_bytes;
new_tokens.push_back(tokens[token_position]);
}
tokens = std::move(new_tokens);
return result;
}
// insert a token of length 1 of the specified type
static void InsertToken(tokenType insert_type, idx_t insert_pos, vector<highlightToken> &tokens) {
vector<highlightToken> new_tokens;
new_tokens.reserve(tokens.size() + 1);
idx_t i;
bool found = false;
for (i = 0; i < tokens.size(); i++) {
// find the exact position where we need to insert the token
if (tokens[i].start == insert_pos) {
// this token is exactly at this render position
// insert highlighting for the bracket
highlightToken token;
token.start = insert_pos;
token.type = insert_type;
token.search_match = false;
new_tokens.push_back(token);
// now we need to insert the other token ONLY if the other token is not immediately following this one
if (i + 1 >= tokens.size() || tokens[i + 1].start > insert_pos + 1) {
token.start = insert_pos + 1;
token.type = tokens[i].type;
token.search_match = false;
new_tokens.push_back(token);
}
i++;
found = true;
break;
} else if (tokens[i].start > insert_pos) {
// the next token is AFTER the render position
// insert highlighting for the bracket
highlightToken token;
token.start = insert_pos;
token.type = insert_type;
token.search_match = false;
new_tokens.push_back(token);
// now just insert the next token
new_tokens.push_back(tokens[i]);
i++;
found = true;
break;
} else {
// insert the token
new_tokens.push_back(tokens[i]);
}
}
// copy over the remaining tokens
for (; i < tokens.size(); i++) {
new_tokens.push_back(tokens[i]);
}
if (!found) {
// token was not added - add it to the end
highlightToken token;
token.start = insert_pos;
token.type = insert_type;
token.search_match = false;
new_tokens.push_back(token);
}
tokens = std::move(new_tokens);
}
enum class ScanState { STANDARD, IN_SINGLE_QUOTE, IN_DOUBLE_QUOTE, IN_COMMENT, DOLLAR_QUOTED_STRING };
static void OpenBracket(vector<idx_t> &brackets, vector<idx_t> &cursor_brackets, idx_t pos, idx_t i) {
// check if the cursor is at this position
if (pos == i) {
// cursor is exactly on this position - always highlight this bracket
if (!cursor_brackets.empty()) {
cursor_brackets.clear();
}
cursor_brackets.push_back(i);
}
if (cursor_brackets.empty() && ((i + 1) == pos || (pos + 1) == i)) {
// cursor is either BEFORE or AFTER this bracket and we don't have any highlighted bracket yet
// highlight this bracket
cursor_brackets.push_back(i);
}
brackets.push_back(i);
}
static void CloseBracket(vector<idx_t> &brackets, vector<idx_t> &cursor_brackets, idx_t pos, idx_t i,
vector<idx_t> &errors) {
if (pos == i) {
// cursor is on this closing bracket
// clear any selected brackets - we always select this one
cursor_brackets.clear();
}
if (brackets.empty()) {
// closing bracket without matching opening bracket
errors.push_back(i);
} else {
if (cursor_brackets.size() == 1) {
if (cursor_brackets.back() == brackets.back()) {
// this closing bracket matches the highlighted opening cursor bracket - highlight both
cursor_brackets.push_back(i);
}
} else if (cursor_brackets.empty() && (pos == i || (i + 1) == pos || (pos + 1) == i)) {
// no cursor bracket selected yet and cursor is BEFORE or AFTER this bracket
// add this bracket
cursor_brackets.push_back(i);
cursor_brackets.push_back(brackets.back());
}
brackets.pop_back();
}
}
static void HandleBracketErrors(const vector<idx_t> &brackets, vector<idx_t> &errors) {
if (brackets.empty()) {
return;
}
// if there are unclosed brackets remaining not all brackets were closed
for (auto &bracket : brackets) {
errors.push_back(bracket);
}
}
void Linenoise::AddErrorHighlighting(idx_t render_start, idx_t render_end, vector<highlightToken> &tokens) const {
static constexpr const idx_t MAX_ERROR_LENGTH = 2000;
if (!enableErrorRendering) {
return;
}
if (len >= MAX_ERROR_LENGTH) {
return;
}
// do a pass over the buffer to collect errors:
// * brackets without matching closing/opening bracket
// * single quotes without matching closing single quote
// * double quote without matching double quote
ScanState state = ScanState::STANDARD;
vector<idx_t> brackets; // ()
vector<idx_t> square_brackets; // []
vector<idx_t> curly_brackets; // {}
vector<idx_t> errors;
vector<idx_t> cursor_brackets;
vector<idx_t> comment_start;
vector<idx_t> comment_end;
string dollar_quote_marker;
idx_t quote_pos = 0;
for (idx_t i = 0; i < len; i++) {
auto c = buf[i];
switch (state) {
case ScanState::STANDARD:
switch (c) {
case '-':
if (i + 1 < len && buf[i + 1] == '-') {
// -- puts us in a comment
comment_start.push_back(i);
i++;
state = ScanState::IN_COMMENT;
break;
}
break;
case '\'':
state = ScanState::IN_SINGLE_QUOTE;
quote_pos = i;
break;
case '\"':
state = ScanState::IN_DOUBLE_QUOTE;
quote_pos = i;
break;
case '(':
OpenBracket(brackets, cursor_brackets, pos, i);
break;
case '[':
OpenBracket(square_brackets, cursor_brackets, pos, i);
break;
case '{':
OpenBracket(curly_brackets, cursor_brackets, pos, i);
break;
case ')':
CloseBracket(brackets, cursor_brackets, pos, i, errors);
break;
case ']':
CloseBracket(square_brackets, cursor_brackets, pos, i, errors);
break;
case '}':
CloseBracket(curly_brackets, cursor_brackets, pos, i, errors);
break;
case '$': { // dollar symbol
if (i + 1 >= len) {
// we need more than just a dollar
break;
}
// check if this is a dollar-quoted string
idx_t next_dollar = 0;
for (idx_t idx = i + 1; idx < len; idx++) {
if (buf[idx] == '$') {
// found the next dollar
next_dollar = idx;
break;
}
// all characters can be between A-Z, a-z or \200 - \377
if (buf[idx] >= 'A' && buf[idx] <= 'Z') {
continue;
}
if (buf[idx] >= 'a' && buf[idx] <= 'z') {
continue;
}
if (buf[idx] >= '\200' && buf[idx] <= '\377') {
continue;
}
// the first character CANNOT be a numeric, only subsequent characters
if (idx > i + 1 && buf[idx] >= '0' && buf[idx] <= '9') {
continue;
}
// not a dollar quoted string
break;
}
if (next_dollar == 0) {
// not a dollar quoted string
break;
}
// dollar quoted string
state = ScanState::DOLLAR_QUOTED_STRING;
quote_pos = i;
i = next_dollar;
if (i < len) {
// found a complete marker - store it
idx_t marker_start = quote_pos + 1;
dollar_quote_marker = string(buf + marker_start, i - marker_start);
}
break;
}
default:
break;
}
break;
case ScanState::IN_COMMENT:
// comment state - the only thing that will get us out is a newline
switch (c) {
case '\r':
case '\n':
// newline - left comment state
state = ScanState::STANDARD;
comment_end.push_back(i);
break;
default:
break;
}
break;
case ScanState::IN_SINGLE_QUOTE:
// single quote - all that will get us out is an unescaped single-quote
if (c == '\'') {
if (i + 1 < len && buf[i + 1] == '\'') {
// double single-quote means the quote is escaped - continue
i++;
break;
} else {
// otherwise revert to standard scan state
state = ScanState::STANDARD;
break;
}
}
break;
case ScanState::IN_DOUBLE_QUOTE:
// double quote - all that will get us out is an unescaped quote
if (c == '"') {
if (i + 1 < len && buf[i + 1] == '"') {
// double quote means the quote is escaped - continue
i++;
break;
} else {
// otherwise revert to standard scan state
state = ScanState::STANDARD;
break;
}
}
break;
case ScanState::DOLLAR_QUOTED_STRING: {
// dollar-quoted string - all that will get us out is a $[marker]$
if (c != '$') {
break;
}
if (i + 1 >= len) {
// no room for the final dollar
break;
}
// skip to the next dollar symbol
idx_t start = i + 1;
idx_t end = start;
while (end < len && buf[end] != '$') {
end++;
}
if (end >= len) {
// no final dollar found - continue as normal
break;
}
if (end - start != dollar_quote_marker.size()) {
// length mismatch - cannot match
break;
}
if (memcmp(buf + start, dollar_quote_marker.c_str(), dollar_quote_marker.size()) != 0) {
// marker mismatch
break;
}
// marker found! revert to standard state
dollar_quote_marker = string();
state = ScanState::STANDARD;
i = end;
break;
}
default:
break;
}
}
if (state == ScanState::IN_DOUBLE_QUOTE || state == ScanState::IN_SINGLE_QUOTE ||
state == ScanState::DOLLAR_QUOTED_STRING) {
// quote is never closed
errors.push_back(quote_pos);
}
HandleBracketErrors(brackets, errors);
HandleBracketErrors(square_brackets, errors);
HandleBracketErrors(curly_brackets, errors);
// insert all the errors for highlighting
for (auto &error : errors) {
Linenoise::Log("Error found at position %llu\n", error);
if (error < render_start || error > render_end) {
continue;
}
auto render_error = error - render_start;
InsertToken(tokenType::TOKEN_ERROR, render_error, tokens);
}
if (cursor_brackets.size() != 2) {
// no matching cursor brackets found
cursor_brackets.clear();
}
// insert bracket for highlighting
for (auto &bracket_position : cursor_brackets) {
Linenoise::Log("Highlight bracket at position %d\n", bracket_position);
if (bracket_position < render_start || bracket_position > render_end) {
continue;
}
idx_t render_position = bracket_position - render_start;
InsertToken(tokenType::TOKEN_BRACKET, render_position, tokens);
}
// insert comments
if (!comment_start.empty()) {
vector<highlightToken> new_tokens;
new_tokens.reserve(tokens.size());
idx_t token_idx = 0;
for (idx_t c = 0; c < comment_start.size(); c++) {
auto c_start = comment_start[c];
auto c_end = c < comment_end.size() ? comment_end[c] : len;
if (c_start < render_start || c_end > render_end) {
continue;
}
Linenoise::Log("Comment at position %d to %d\n", c_start, c_end);
c_start -= render_start;
c_end -= render_start;
bool inserted_comment = false;
highlightToken comment_token;
comment_token.start = c_start;
comment_token.type = tokenType::TOKEN_COMMENT;
comment_token.search_match = false;
for (; token_idx < tokens.size(); token_idx++) {
if (tokens[token_idx].start >= c_start) {
// insert the comment here
new_tokens.push_back(comment_token);
inserted_comment = true;
break;
}
new_tokens.push_back(tokens[token_idx]);
}
if (!inserted_comment) {
new_tokens.push_back(comment_token);
} else {
// skip all tokens until we exit the comment again
for (; token_idx < tokens.size(); token_idx++) {
if (tokens[token_idx].start > c_end) {
break;
}
}
}
}
for (; token_idx < tokens.size(); token_idx++) {
new_tokens.push_back(tokens[token_idx]);
}
tokens = std::move(new_tokens);
}
}
static bool IsCompletionCharacter(char c) {
if (c >= 'A' && c <= 'Z') {
return true;
}
if (c >= 'a' && c <= 'z') {
return true;
}
if (c == '_') {
return true;
}
return false;
}
bool Linenoise::AddCompletionMarker(const char *buf, idx_t len, string &result_buffer,
vector<highlightToken> &tokens) const {
if (!enableCompletionRendering) {
return false;
}
if (!continuation_markers) {
// don't render when pressing ctrl+c, only when editing
return false;
}
static constexpr const idx_t MAX_COMPLETION_LENGTH = 1000;
if (len >= MAX_COMPLETION_LENGTH) {
return false;
}
if (!insert || pos != len) {
// only show when inserting a character at the end
return false;
}
if (pos < 3) {
// we need at least 3 bytes
return false;
}
if (!tokens.empty() && tokens.back().type == tokenType::TOKEN_ERROR) {
// don't show auto-completion when we have errors
return false;
}
// we ONLY show completion if we have typed at least three characters that are supported for completion
// for now this is ONLY the characters a-z, A-Z and underscore (_)
for (idx_t i = pos - 3; i < pos; i++) {
if (!IsCompletionCharacter(buf[i])) {
return false;
}
}
auto completion = TabComplete();
if (completion.completions.empty()) {
// no completions found
return false;
}
if (completion.completions[0].completion.size() <= len) {
// completion is not long enough
return false;
}
// we have stricter requirements for rendering completions - the completion must match exactly
for (idx_t i = pos; i > 0; i--) {
auto cpos = i - 1;
if (!IsCompletionCharacter(buf[cpos])) {
break;
}
if (completion.completions[0].completion[cpos] != buf[cpos]) {
return false;
}
}
// add the first completion found for rendering purposes
result_buffer = string(buf, len);
result_buffer += completion.completions[0].completion.substr(len);
highlightToken completion_token;
completion_token.start = len;
completion_token.type = tokenType::TOKEN_COMMENT;
completion_token.search_match = true;
tokens.push_back(completion_token);
return true;
}
/* Multi line low level line refresh.
*
* Rewrite the currently edited line accordingly to the buffer content,
* cursor position, and number of columns of the terminal. */
void Linenoise::RefreshMultiLine() {
if (!render) {
return;
}
char seq[64];
int plen = GetPromptWidth();
// utf8 in prompt, get render width
int rows, cols;
int new_cursor_row, new_cursor_x;
PositionToColAndRow(pos, new_cursor_row, new_cursor_x, rows, cols);
int col; /* column position, zero-based. */
int old_rows = maxrows ? maxrows : 1;
int fd = ofd;
std::string highlight_buffer;
auto render_buf = this->buf;
auto render_len = this->len;
idx_t render_start = 0;
idx_t render_end = render_len;
if (clear_screen) {
old_cursor_rows = 0;
old_rows = 0;
clear_screen = false;
}
if (rows > ws.ws_row) {
// the text does not fit in the terminal (too many rows)
// enable scrolling mode
// check if, given the current y_scroll, the cursor is visible
// display range is [y_scroll, y_scroll + ws.ws_row]
if (new_cursor_row < int(y_scroll) + 1) {
y_scroll = new_cursor_row - 1;
} else if (new_cursor_row > int(y_scroll) + int(ws.ws_row)) {
y_scroll = new_cursor_row - ws.ws_row;
}
// display only characters up to the current scroll position
if (y_scroll == 0) {
render_start = 0;
} else {
render_start = ColAndRowToPosition(y_scroll, 0);
}
if (int(y_scroll) + int(ws.ws_row) >= rows) {
render_end = len;
} else {
render_end = ColAndRowToPosition(y_scroll + ws.ws_row, 99999);
}
new_cursor_row -= y_scroll;
render_buf += render_start;
render_len = render_end - render_start;
Linenoise::Log("truncate to rows %d - %d (render bytes %d to %d)", y_scroll, y_scroll + ws.ws_row, render_start,
render_end);
rows = ws.ws_row;
} else {
y_scroll = 0;
}
/* Update maxrows if needed. */
if (rows > (int)maxrows) {
maxrows = rows;
}
vector<highlightToken> tokens;
if (Highlighting::IsEnabled()) {
bool is_dot_command = buf[0] == '.';
auto match = search_index < search_matches.size() ? &search_matches[search_index] : nullptr;
tokens = Highlighting::Tokenize(render_buf, render_len, is_dot_command, match);
// add error highlighting
AddErrorHighlighting(render_start, render_end, tokens);
// add completion hint
if (AddCompletionMarker(render_buf, render_len, highlight_buffer, tokens)) {
render_buf = (char *)highlight_buffer.c_str();
render_len = highlight_buffer.size();
}
}
if (rows > 1) {
// add continuation markers
highlight_buffer = AddContinuationMarkers(render_buf, render_len, plen,
y_scroll > 0 ? new_cursor_row + 1 : new_cursor_row, tokens);
render_buf = (char *)highlight_buffer.c_str();
render_len = highlight_buffer.size();
}
if (duckdb::Utf8Proc::IsValid(render_buf, render_len)) {
if (Highlighting::IsEnabled()) {
highlight_buffer = Highlighting::HighlightText(render_buf, render_len, 0, render_len, tokens);
render_buf = (char *)highlight_buffer.c_str();
render_len = highlight_buffer.size();
}
}
/* First step: clear all the lines used before. To do so start by
* going to the last row. */
AppendBuffer append_buffer;
if (old_rows - old_cursor_rows > 0) {
Linenoise::Log("go down %d", old_rows - old_cursor_rows);
snprintf(seq, 64, "\x1b[%dB", old_rows - int(old_cursor_rows));
append_buffer.Append(seq);
}
/* Now for every row clear it, go up. */
for (int j = 0; j < old_rows - 1; j++) {
Linenoise::Log("clear+up");
append_buffer.Append("\r\x1b[0K\x1b[1A");
}
/* Clean the top line. */
Linenoise::Log("clear");
append_buffer.Append("\r\x1b[0K");
/* Write the prompt and the current buffer content */
if (y_scroll == 0) {
append_buffer.Append(prompt);
}
append_buffer.Append(render_buf, render_len);
/* Show hints if any. */
RefreshShowHints(append_buffer, plen);
/* If we are at the very end of the screen with our prompt, we need to
* emit a newline and move the prompt to the first column. */
Linenoise::Log("pos > 0 %d", pos > 0 ? 1 : 0);
Linenoise::Log("pos == len %d", pos == len ? 1 : 0);
Linenoise::Log("new_cursor_x == cols %d", new_cursor_x == ws.ws_col ? 1 : 0);
if (pos > 0 && pos == len && new_cursor_x == ws.ws_col) {
Linenoise::Log("<newline>", 0);
append_buffer.Append("\n");
append_buffer.Append("\r");
rows++;
new_cursor_row++;
new_cursor_x = 0;
if (rows > (int)maxrows) {
maxrows = rows;
}
}
Linenoise::Log("render %d rows (old rows %d)", rows, old_rows);
/* Move cursor to right position. */
Linenoise::Log("new_cursor_row %d", new_cursor_row);
Linenoise::Log("new_cursor_x %d", new_cursor_x);
Linenoise::Log("len %d", len);
Linenoise::Log("old_cursor_rows %d", old_cursor_rows);
Linenoise::Log("pos %d", pos);
Linenoise::Log("max cols %d", ws.ws_col);
/* Go up till we reach the expected position. */
if (rows - new_cursor_row > 0) {
Linenoise::Log("go-up %d", rows - new_cursor_row);
snprintf(seq, 64, "\x1b[%dA", rows - new_cursor_row);
append_buffer.Append(seq);
}
/* Set column. */
col = new_cursor_x;
Linenoise::Log("set col %d", 1 + col);
if (col) {
snprintf(seq, 64, "\r\x1b[%dC", col);
} else {
snprintf(seq, 64, "\r");
}
append_buffer.Append(seq);
Linenoise::Log("\n");
old_cursor_rows = new_cursor_row;
append_buffer.Write(fd);
}
/* Calls the two low level functions refreshSingleLine() or
* refreshMultiLine() according to the selected mode. */
void Linenoise::RefreshLine() {
if (Terminal::IsMultiline()) {
RefreshMultiLine();
} else {
RefreshSingleLine();
}
}
} // namespace duckdb

496
external/duckdb/tools/shell/linenoise/terminal.cpp vendored Normal file
View File

@@ -0,0 +1,496 @@
#include "terminal.hpp"
#include "history.hpp"
#include "linenoise.hpp"
#include <termios.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>
#include <stdlib.h>
#include <ctype.h>
#include <sys/ioctl.h>
#include <sys/select.h>
#include <sys/time.h>
namespace duckdb {
static int mlmode = 1; /* Multi line mode. Default is multi line. */
static struct termios orig_termios; /* In order to restore at exit.*/
static int atexit_registered = 0; /* Register atexit just 1 time. */
static int rawmode = 0; /* For atexit() function to check if restore is needed*/
static const char *unsupported_term[] = {"dumb", "cons25", "emacs", NULL};
/* At exit we'll try to fix the terminal to the initial conditions. */
static void linenoiseAtExit(void) {
Terminal::DisableRawMode();
History::Free();
}
/* Return true if the terminal name is in the list of terminals we know are
* not able to understand basic escape sequences. */
int Terminal::IsUnsupportedTerm() {
char *term = getenv("TERM");
int j;
if (!term) {
return 0;
}
for (j = 0; unsupported_term[j]; j++) {
if (!strcasecmp(term, unsupported_term[j])) {
return 1;
}
}
return 0;
}
/* Raw mode: 1960 magic shit. */
int Terminal::EnableRawMode() {
int fd = STDIN_FILENO;
if (!isatty(STDIN_FILENO)) {
errno = ENOTTY;
return -1;
}
if (!atexit_registered) {
atexit(linenoiseAtExit);
atexit_registered = 1;
}
if (tcgetattr(fd, &orig_termios) == -1) {
errno = ENOTTY;
return -1;
}
auto raw = orig_termios; /* modify the original mode */
/* input modes: no break, no CR to NL, no parity check, no strip char,
* no start/stop output control. */
raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON);
/* output modes - disable post processing */
raw.c_oflag &= ~(OPOST);
#ifdef IUTF8
/* control modes - set 8 bit chars */
raw.c_iflag |= IUTF8;
#endif
raw.c_cflag |= CS8;
/* local modes - choing off, canonical off, no extended functions,
* no signal chars (^Z,^C) */
raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG);
/* control chars - set return condition: min number of bytes and timer.
* We want read to return every single byte, without timeout. */
raw.c_cc[VMIN] = 1;
raw.c_cc[VTIME] = 0; /* 1 byte, no timer */
/* put terminal in raw mode after flushing */
if (tcsetattr(fd, TCSADRAIN, &raw) < 0) {
errno = ENOTTY;
return -1;
}
rawmode = 1;
return 0;
}
void Terminal::DisableRawMode() {
int fd = STDIN_FILENO;
/* Don't even check the return value as it's too late. */
if (rawmode && tcsetattr(fd, TCSADRAIN, &orig_termios) != -1) {
rawmode = 0;
}
}
bool Terminal::IsMultiline() {
return mlmode;
}
bool Terminal::IsAtty() {
return isatty(STDIN_FILENO);
}
/* This function is called when linenoise() is called with the standard
* input file descriptor not attached to a TTY. So for example when the
* program using linenoise is called in pipe or with a file redirected
* to its standard input. In this case, we want to be able to return the
* line regardless of its length (by default we are limited to 4k). */
char *Terminal::EditNoTTY() {
char *line = NULL;
size_t len = 0, maxlen = 0;
while (1) {
if (len == maxlen) {
if (maxlen == 0)
maxlen = 16;
maxlen *= 2;
char *oldval = line;
line = (char *)realloc(line, maxlen);
if (line == NULL) {
if (oldval)
free(oldval);
return NULL;
}
}
int c = fgetc(stdin);
if (c == EOF || c == '\n') {
if (c == EOF && len == 0) {
free(line);
return NULL;
} else {
line[len] = '\0';
return line;
}
} else {
line[len] = c;
len++;
}
}
}
/* This function calls the line editing function linenoiseEdit() using
* the STDIN file descriptor set in raw mode. */
int Terminal::EditRaw(char *buf, size_t buflen, const char *prompt) {
int count;
if (buflen == 0) {
errno = EINVAL;
return -1;
}
if (Terminal::EnableRawMode() == -1) {
return -1;
}
Linenoise l(STDIN_FILENO, STDOUT_FILENO, buf, buflen, prompt);
count = l.Edit();
Terminal::DisableRawMode();
printf("\n");
return count;
}
// returns true if there is more data available to read in a particular stream
int Terminal::HasMoreData(int fd) {
fd_set rfds;
FD_ZERO(&rfds);
FD_SET(fd, &rfds);
// no timeout: return immediately
struct timeval tv;
tv.tv_sec = 0;
tv.tv_usec = 0;
return select(1, &rfds, NULL, NULL, &tv);
}
/* ======================= Low level terminal handling ====================== */
/* Set if to use or not the multi line mode. */
void Terminal::SetMultiLine(int ml) {
mlmode = ml;
}
static int parseInt(const char *s, int *offset = nullptr) {
int result = 0;
int idx;
for (idx = 0; s[idx]; idx++) {
char c = s[idx];
if (c < '0' || c > '9') {
break;
}
result = result * 10 + c - '0';
if (result > 1000000) {
result = 1000000;
}
}
if (offset) {
*offset = idx;
}
return result;
}
static int tryParseEnv(const char *env_var) {
char *s;
s = getenv(env_var);
if (!s) {
return 0;
}
return parseInt(s);
}
/* Use the ESC [6n escape sequence to query the cursor position
* and return it. On error -1 is returned, on success the position of the
* cursor. */
TerminalSize Terminal::GetCursorPosition() {
int ifd = STDIN_FILENO;
int ofd = STDOUT_FILENO;
TerminalSize ws;
char buf[32];
unsigned int i = 0;
/* Report cursor location */
if (write(ofd, "\x1b[6n", 4) != 4) {
return ws;
}
/* Read the response: ESC [ rows ; cols R */
while (i < sizeof(buf) - 1) {
if (read(ifd, buf + i, 1) != 1) {
break;
}
if (buf[i] == 'R') {
break;
}
i++;
}
buf[i] = '\0';
/* Parse it. */
if (buf[0] != ESC || buf[1] != '[') {
return ws;
}
int offset = 2;
int new_offset;
ws.ws_row = parseInt(buf + offset, &new_offset);
offset += new_offset;
if (buf[offset] != ';') {
return ws;
}
offset++;
ws.ws_col = parseInt(buf + offset);
return ws;
}
TerminalSize Terminal::TryMeasureTerminalSize() {
int ofd = STDOUT_FILENO;
/* ioctl() failed. Try to query the terminal itself. */
TerminalSize start, result;
/* Get the initial position so we can restore it later. */
start = GetCursorPosition();
if (!start.ws_col) {
return result;
}
/* Go to bottom-right margin */
if (write(ofd, "\x1b[999;999f", 10) != 10) {
return result;
}
result = GetCursorPosition();
if (!result.ws_col) {
return result;
}
/* Restore position. */
char seq[32];
snprintf(seq, 32, "\x1b[%d;%df", start.ws_row, start.ws_col);
if (write(ofd, seq, strlen(seq)) == -1) {
/* Can't recover... */
}
return result;
}
/* Try to get the number of columns in the current terminal, or assume 80
* if it fails. */
TerminalSize Terminal::GetTerminalSize() {
TerminalSize result;
// try ioctl first
{
struct winsize ws;
ioctl(1, TIOCGWINSZ, &ws);
result.ws_col = ws.ws_col;
result.ws_row = ws.ws_row;
}
// try ROWS and COLUMNS env variables
if (!result.ws_col) {
result.ws_col = tryParseEnv("COLUMNS");
}
if (!result.ws_row) {
result.ws_row = tryParseEnv("ROWS");
}
// if those fail measure the size by moving the cursor to the corner and fetching the position
if (!result.ws_col || !result.ws_row) {
TerminalSize measured_size = TryMeasureTerminalSize();
Linenoise::Log("measured size col %d,row %d -- ", measured_size.ws_row, measured_size.ws_col);
if (measured_size.ws_row) {
result.ws_row = measured_size.ws_row;
}
if (measured_size.ws_col) {
result.ws_col = measured_size.ws_col;
}
}
// if all else fails use defaults (80,24)
if (!result.ws_col) {
result.ws_col = 80;
}
if (!result.ws_row) {
result.ws_row = 24;
}
return result;
}
/* Clear the screen. Used to handle ctrl+l */
void Terminal::ClearScreen() {
if (write(STDOUT_FILENO, "\x1b[H\x1b[2J", 7) <= 0) {
/* nothing to do, just to avoid warning. */
}
}
/* Beep, used for completion when there is nothing to complete or when all
* the choices were already shown. */
void Terminal::Beep() {
fprintf(stderr, "\x7");
fflush(stderr);
}
EscapeSequence Terminal::ReadEscapeSequence(int ifd) {
char seq[5];
idx_t length = ReadEscapeSequence(ifd, seq);
if (length == 0) {
return EscapeSequence::INVALID;
}
Linenoise::Log("escape of length %d\n", length);
switch (length) {
case 1:
if (seq[0] >= 'a' && seq[0] <= 'z') {
return EscapeSequence(idx_t(EscapeSequence::ALT_A) + (seq[0] - 'a'));
}
if (seq[0] >= 'A' && seq[0] <= 'Z') {
return EscapeSequence(idx_t(EscapeSequence::ALT_A) + (seq[0] - 'A'));
}
switch (seq[0]) {
case BACKSPACE:
return EscapeSequence::ALT_BACKSPACE;
case ESC:
return EscapeSequence::ESCAPE;
case '<':
return EscapeSequence::ALT_LEFT_ARROW;
case '>':
return EscapeSequence::ALT_RIGHT_ARROW;
case '\\':
return EscapeSequence::ALT_BACKSLASH;
default:
Linenoise::Log("unrecognized escape sequence of length 1 - %d\n", seq[0]);
break;
}
break;
case 2:
if (seq[0] == 'O') {
switch (seq[1]) {
case 'A': /* Up */
return EscapeSequence::UP;
case 'B': /* Down */
return EscapeSequence::DOWN;
case 'C': /* Right */
return EscapeSequence::RIGHT;
case 'D': /* Left */
return EscapeSequence::LEFT;
case 'H': /* Home */
return EscapeSequence::HOME;
case 'F': /* End*/
return EscapeSequence::END;
case 'c':
return EscapeSequence::ALT_F;
case 'd':
return EscapeSequence::ALT_B;
default:
Linenoise::Log("unrecognized escape sequence (O) %d\n", seq[1]);
break;
}
} else if (seq[0] == '[') {
switch (seq[1]) {
case 'A': /* Up */
return EscapeSequence::UP;
case 'B': /* Down */
return EscapeSequence::DOWN;
case 'C': /* Right */
return EscapeSequence::RIGHT;
case 'D': /* Left */
return EscapeSequence::LEFT;
case 'H': /* Home */
return EscapeSequence::HOME;
case 'F': /* End*/
return EscapeSequence::END;
case 'Z': /* Shift Tab */
return EscapeSequence::SHIFT_TAB;
default:
Linenoise::Log("unrecognized escape sequence (seq[1]) %d\n", seq[1]);
break;
}
} else {
Linenoise::Log("unrecognized escape sequence of length %d (%d %d)\n", length, seq[0], seq[1]);
}
break;
case 3:
if (seq[2] == '~') {
switch (seq[1]) {
case '1':
return EscapeSequence::HOME;
case '3': /* Delete key. */
return EscapeSequence::DELETE;
case '4':
case '8':
return EscapeSequence::END;
default:
Linenoise::Log("unrecognized escape sequence (~) %d\n", seq[1]);
break;
}
} else if (seq[1] == '5' && seq[2] == 'C') {
return EscapeSequence::ALT_F;
} else if (seq[1] == '5' && seq[2] == 'D') {
return EscapeSequence::ALT_B;
} else {
Linenoise::Log("unrecognized escape sequence of length %d\n", length);
}
break;
case 5:
if (memcmp(seq, "[1;5C", 5) == 0 || memcmp(seq, "[1;3C", 5) == 0) {
// [1;5C: move word right
return EscapeSequence::CTRL_MOVE_FORWARDS;
} else if (memcmp(seq, "[1;5D", 5) == 0 || memcmp(seq, "[1;3D", 5) == 0) {
// [1;5D: move word left
return EscapeSequence::CTRL_MOVE_BACKWARDS;
} else {
Linenoise::Log("unrecognized escape sequence (;) %d\n", seq[1]);
}
break;
default:
Linenoise::Log("unrecognized escape sequence of length %d\n", length);
break;
}
return EscapeSequence::UNKNOWN;
}
idx_t Terminal::ReadEscapeSequence(int ifd, char seq[]) {
if (read(ifd, seq, 1) == -1) {
return 0;
}
switch (seq[0]) {
case 'O':
case '[':
// these characters have multiple bytes following them
break;
default:
return 1;
}
if (read(ifd, seq + 1, 1) == -1) {
return 0;
}
if (seq[0] != '[') {
return 2;
}
if (seq[1] < '0' || seq[1] > '9') {
return 2;
}
/* Extended escape, read additional byte. */
if (read(ifd, seq + 2, 1) == -1) {
return 0;
}
if (seq[2] == ';') {
// read 2 extra bytes
if (read(ifd, seq + 3, 2) == -1) {
return 0;
}
return 5;
} else {
return 3;
}
}
} // namespace duckdb

40
external/duckdb/tools/shell/rc/duckdb.rc vendored Normal file
View File

@@ -0,0 +1,40 @@
#include <windows.h>
#define Q(x) #x
#define QUOTE(x) Q(x)
VS_VERSION_INFO VERSIONINFO
FILEVERSION DUCKDB_MAJOR_VERSION,DUCKDB_MINOR_VERSION,DUCKDB_PATCH_VERSION,DUCKDB_DEV_ITERATION
PRODUCTVERSION DUCKDB_MAJOR_VERSION,DUCKDB_MINOR_VERSION,DUCKDB_PATCH_VERSION,DUCKDB_DEV_ITERATION
#ifdef DEBUG
FILEFLAGSMASK VS_FF_DEBUG | VS_FF_PRERELEASE
#else
FILEFLAGSMASK 0
#endif
FILEOS VOS_NT_WINDOWS32
FILETYPE VFT_APP
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "Comments", "DuckDB shell"
VALUE "CompanyName", "DuckDB Labs"
VALUE "FileDescription", "DuckDB shell"
VALUE "FileVersion", QUOTE(DUCKDB_VERSION)
VALUE "InternalName", "DuckDB shell"
VALUE "LegalCopyright", "Copyright 2018-" QUOTE(DUCKDB_COPYRIGHT_YEAR) " Stichting DuckDB Foundation"
VALUE "OriginalFilename", "duckdb.exe"
VALUE "ProductName", "DuckDB"
VALUE "ProductVersion", QUOTE(DUCKDB_VERSION)
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1252
END
END
MAINICON ICON "../../../logo/DuckDB.ico"

5357
external/duckdb/tools/shell/shell.cpp vendored Normal file
View File

File diff suppressed because it is too large Load Diff

294
external/duckdb/tools/shell/shell_highlight.cpp vendored Normal file
View File

@@ -0,0 +1,294 @@
#include "shell_highlight.hpp"
#include "shell_state.hpp"
#include "duckdb/parser/parser.hpp"
#if defined(_WIN32) || defined(WIN32)
#include <windows.h>
#endif
namespace duckdb_shell {
struct HighlightElement {
const char *name;
PrintColor color;
PrintIntensity intensity;
};
static HighlightElement highlight_elements[] = {{"error", PrintColor::RED, PrintIntensity::BOLD},
{"keyword", PrintColor::GREEN, PrintIntensity::STANDARD},
{"numeric_constant", PrintColor::YELLOW, PrintIntensity::STANDARD},
{"string_constant", PrintColor::YELLOW, PrintIntensity::STANDARD},
{"line_indicator", PrintColor::STANDARD, PrintIntensity::BOLD},
{"column_name", PrintColor::STANDARD, PrintIntensity::STANDARD},
{"column_type", PrintColor::STANDARD, PrintIntensity::STANDARD},
{"numeric_value", PrintColor::STANDARD, PrintIntensity::STANDARD},
{"string_value", PrintColor::STANDARD, PrintIntensity::STANDARD},
{"temporal_value", PrintColor::STANDARD, PrintIntensity::STANDARD},
{"null_value", PrintColor::GRAY, PrintIntensity::STANDARD},
{"footer", PrintColor::STANDARD, PrintIntensity::STANDARD},
{"layout", PrintColor::GRAY, PrintIntensity::STANDARD},
{"none", PrintColor::STANDARD, PrintIntensity::STANDARD},
{nullptr, PrintColor::STANDARD, PrintIntensity::STANDARD}};
struct HighlightColors {
const char *name;
PrintColor color;
};
static const HighlightColors highlight_colors[] = {{"standard", PrintColor::STANDARD}, {"red", PrintColor::RED},
{"yellow", PrintColor::YELLOW}, {"green", PrintColor::GREEN},
{"gray", PrintColor::GRAY}, {"blue", PrintColor::BLUE},
{"magenta", PrintColor::MAGENTA}, {"cyan", PrintColor::CYAN},
{"white", PrintColor::WHITE}, {nullptr, PrintColor::STANDARD}};
ShellHighlight::ShellHighlight(ShellState &state) : state(state) {
}
/*
** Output text to the console in a font that attracts extra attention.
*/
#ifdef _WIN32
void ShellHighlight::PrintText(const string &text, PrintOutput output, PrintColor color, PrintIntensity intensity) {
HANDLE out = GetStdHandle(output == PrintOutput::STDOUT ? STD_OUTPUT_HANDLE : STD_ERROR_HANDLE);
CONSOLE_SCREEN_BUFFER_INFO defaultScreenInfo;
GetConsoleScreenBufferInfo(out, &defaultScreenInfo);
WORD wAttributes = 0;
switch (intensity) {
case PrintIntensity::BOLD:
case PrintIntensity::BOLD_UNDERLINE:
wAttributes |= FOREGROUND_INTENSITY;
break;
default:
break;
}
switch (color) {
case PrintColor::RED:
wAttributes |= FOREGROUND_RED;
break;
case PrintColor::GREEN:
wAttributes |= FOREGROUND_GREEN;
break;
case PrintColor::BLUE:
wAttributes |= FOREGROUND_BLUE;
break;
case PrintColor::YELLOW:
wAttributes |= FOREGROUND_RED | FOREGROUND_GREEN;
break;
case PrintColor::GRAY:
wAttributes |= FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE;
break;
case PrintColor::MAGENTA:
wAttributes |= FOREGROUND_BLUE | FOREGROUND_RED;
break;
case PrintColor::CYAN:
wAttributes |= FOREGROUND_BLUE | FOREGROUND_GREEN;
break;
case PrintColor::WHITE:
wAttributes |= FOREGROUND_BLUE | FOREGROUND_GREEN | FOREGROUND_RED;
break;
default:
break;
}
if (wAttributes != 0) {
SetConsoleTextAttribute(out, wAttributes);
}
state.Print(output, text);
SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes);
}
#else
void ShellHighlight::PrintText(const string &text, PrintOutput output, PrintColor color, PrintIntensity intensity) {
const char *bold_prefix = "";
const char *color_prefix = "";
const char *suffix = "";
switch (intensity) {
case PrintIntensity::BOLD:
bold_prefix = "\033[1m";
break;
case PrintIntensity::UNDERLINE:
bold_prefix = "\033[4m";
break;
case PrintIntensity::BOLD_UNDERLINE:
bold_prefix = "\033[1m\033[4m";
break;
default:
break;
}
switch (color) {
case PrintColor::RED:
color_prefix = "\033[31m";
break;
case PrintColor::GREEN:
color_prefix = "\033[32m";
break;
case PrintColor::YELLOW:
color_prefix = "\033[33m";
break;
case PrintColor::GRAY:
color_prefix = "\033[90m";
break;
case PrintColor::BLUE:
color_prefix = "\033[34m";
break;
case PrintColor::MAGENTA:
color_prefix = "\033[35m";
break;
case PrintColor::CYAN:
color_prefix = "\033[36m";
break;
case PrintColor::WHITE:
color_prefix = "\033[37m";
break;
default:
break;
}
if (*color_prefix || *bold_prefix) {
suffix = "\033[0m";
}
fprintf(output == PrintOutput::STDOUT ? state.out : stderr, "%s%s%s%s", bold_prefix, color_prefix, text.c_str(),
suffix);
}
#endif
void ShellHighlight::PrintText(const string &text, PrintOutput output, HighlightElementType type) {
auto index = static_cast<uint32_t>(type);
auto max_index = static_cast<uint32_t>(HighlightElementType::NONE);
if (index > max_index) {
index = max_index;
}
auto highlight_info = highlight_elements[index];
PrintText(text, output, highlight_info.color, highlight_info.intensity);
}
void ShellHighlight::PrintError(string error_msg) {
if (error_msg.empty()) {
return;
}
vector<duckdb::SimplifiedToken> tokens;
string error_type;
auto error_location = duckdb::StringUtil::Find(error_msg, "Error: ");
if (error_location.IsValid()) {
error_type = error_msg.substr(0, error_location.GetIndex() + 6);
error_msg = error_msg.substr(error_location.GetIndex() + 7);
}
try {
tokens = duckdb::Parser::TokenizeError(error_msg);
} catch (...) {
// fallback
state.Print(PrintOutput::STDERR, error_msg.c_str());
state.Print(PrintOutput::STDERR, "\n");
return;
}
if (!tokens.empty() && tokens[0].start > 0) {
duckdb::SimplifiedToken new_token;
new_token.type = duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_IDENTIFIER;
new_token.start = 0;
tokens.insert(tokens.begin(), new_token);
}
if (tokens.empty() && !error_msg.empty()) {
duckdb::SimplifiedToken new_token;
new_token.type = duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_IDENTIFIER;
new_token.start = 0;
tokens.push_back(new_token);
}
if (!error_type.empty()) {
PrintText(error_type + "\n", PrintOutput::STDERR, HighlightElementType::ERROR_TOKEN);
}
for (idx_t i = 0; i < tokens.size(); i++) {
HighlightElementType element_type = HighlightElementType::NONE;
switch (tokens[i].type) {
case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_IDENTIFIER:
break;
case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_ERROR:
element_type = HighlightElementType::ERROR_TOKEN;
break;
case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_NUMERIC_CONSTANT:
element_type = HighlightElementType::NUMERIC_CONSTANT;
break;
case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_STRING_CONSTANT:
element_type = HighlightElementType::STRING_CONSTANT;
break;
case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_OPERATOR:
break;
case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_KEYWORD:
element_type = HighlightElementType::KEYWORD;
break;
case duckdb::SimplifiedTokenType::SIMPLIFIED_TOKEN_COMMENT:
element_type = HighlightElementType::LINE_INDICATOR;
break;
}
idx_t start = tokens[i].start;
idx_t end = i + 1 == tokens.size() ? error_msg.size() : tokens[i + 1].start;
if (end - start > 0) {
string error_print = error_msg.substr(tokens[i].start, end - start);
PrintText(error_print, PrintOutput::STDERR, element_type);
}
}
PrintText("\n", PrintOutput::STDERR, PrintColor::STANDARD, PrintIntensity::STANDARD);
}
bool ShellHighlight::SetColor(const char *element_type, const char *color, const char *intensity) {
idx_t i;
for (i = 0; highlight_elements[i].name; i++) {
if (duckdb::StringUtil::CIEquals(element_type, highlight_elements[i].name)) {
break;
}
}
if (!highlight_elements[i].name) {
// element not found
string supported_options;
for (i = 0; highlight_elements[i].name; i++) {
if (!supported_options.empty()) {
supported_options += ", ";
}
supported_options += highlight_elements[i].name;
}
state.Print(PrintOutput::STDERR, duckdb::StringUtil::Format("Unknown element '%s', supported options: %s\n",
element_type, supported_options.c_str()));
return false;
}
// found the element - parse the color
idx_t c;
for (c = 0; highlight_colors[c].name; c++) {
if (duckdb::StringUtil::CIEquals(color, highlight_colors[c].name)) {
break;
}
}
if (!highlight_colors[c].name) {
// color not found
string supported_options;
for (c = 0; highlight_colors[c].name; c++) {
if (!supported_options.empty()) {
supported_options += ", ";
}
supported_options += highlight_colors[c].name;
}
state.Print(PrintOutput::STDERR, duckdb::StringUtil::Format("Unknown color '%s', supported options: %s\n",
color, supported_options.c_str()));
return false;
}
highlight_elements[i].color = highlight_colors[c].color;
highlight_elements[i].intensity = PrintIntensity::STANDARD;
if (intensity) {
if (duckdb::StringUtil::CIEquals(intensity, "standard")) {
highlight_elements[i].intensity = PrintIntensity::STANDARD;
} else if (duckdb::StringUtil::CIEquals(intensity, "bold")) {
highlight_elements[i].intensity = PrintIntensity::BOLD;
} else if (duckdb::StringUtil::CIEquals(intensity, "underline")) {
highlight_elements[i].intensity = PrintIntensity::UNDERLINE;
} else if (duckdb::StringUtil::CIEquals(intensity, "bold_underline")) {
highlight_elements[i].intensity = PrintIntensity::BOLD_UNDERLINE;
} else {
state.Print(PrintOutput::STDERR,
duckdb::StringUtil::Format(
"Unknown intensity '%s', supported options: standard, bold, underline\n", intensity));
return false;
}
}
return true;
}
} // namespace duckdb_shell

844
external/duckdb/tools/shell/shell_renderer.cpp vendored Normal file
View File

@@ -0,0 +1,844 @@
#include "shell_renderer.hpp"
#include "shell_state.hpp"
#include "duckdb_shell_wrapper.h"
#include "sqlite3.h"
#include <stdexcept>
#include <cstring>
namespace duckdb_shell {
bool ShellRenderer::IsColumnar(RenderMode mode) {
switch (mode) {
case RenderMode::COLUMN:
case RenderMode::TABLE:
case RenderMode::BOX:
case RenderMode::MARKDOWN:
case RenderMode::LATEX:
return true;
default:
return false;
}
}
ShellRenderer::ShellRenderer(ShellState &state)
: state(state), show_header(state.showHeader), col_sep(state.colSeparator), row_sep(state.rowSeparator) {
}
//===--------------------------------------------------------------------===//
// Column Renderers
//===--------------------------------------------------------------------===//
ColumnRenderer::ColumnRenderer(ShellState &state) : ShellRenderer(state) {
}
void ColumnRenderer::RenderFooter(ColumnarResult &result) {
}
void ColumnRenderer::RenderAlignedValue(ColumnarResult &result, idx_t i) {
idx_t w = result.column_width[i];
idx_t n = state.RenderLength(result.data[i]);
state.PrintPadded("", (w - n) / 2);
state.Print(result.data[i]);
state.PrintPadded("", (w - n + 1) / 2);
}
class ModeColumnRenderer : public ColumnRenderer {
public:
explicit ModeColumnRenderer(ShellState &state) : ColumnRenderer(state) {
}
void RenderHeader(ColumnarResult &result) override {
if (!show_header) {
return;
}
for (idx_t i = 0; i < result.column_count; i++) {
state.UTF8WidthPrint(state.out, result.column_width[i], result.data[i], result.right_align[i]);
state.Print(i == result.column_count - 1 ? "\n" : " ");
}
for (idx_t i = 0; i < result.column_count; i++) {
state.PrintDashes(result.column_width[i]);
state.Print(i == result.column_count - 1 ? "\n" : " ");
}
}
const char *GetColumnSeparator() override {
return " ";
}
const char *GetRowSeparator() override {
return "\n";
}
};
class ModeTableRenderer : public ColumnRenderer {
public:
explicit ModeTableRenderer(ShellState &state) : ColumnRenderer(state) {
}
void RenderHeader(ColumnarResult &result) override {
state.PrintRowSeparator(result.column_count, "+", result.column_width);
state.Print("| ");
for (idx_t i = 0; i < result.column_count; i++) {
RenderAlignedValue(result, i);
state.Print(i == result.column_count - 1 ? " |\n" : " | ");
}
state.PrintRowSeparator(result.column_count, "+", result.column_width);
}
void RenderFooter(ColumnarResult &result) override {
state.PrintRowSeparator(result.column_count, "+", result.column_width);
}
const char *GetColumnSeparator() override {
return " | ";
}
const char *GetRowSeparator() override {
return " |\n";
}
const char *GetRowStart() override {
return "| ";
}
};
class ModeMarkdownRenderer : public ColumnRenderer {
public:
explicit ModeMarkdownRenderer(ShellState &state) : ColumnRenderer(state) {
}
void RenderHeader(ColumnarResult &result) override {
state.Print(GetRowStart());
for (idx_t i = 0; i < result.column_count; i++) {
if (i > 0) {
state.Print(GetColumnSeparator());
}
RenderAlignedValue(result, i);
}
state.Print(GetRowSeparator());
state.PrintMarkdownSeparator(result.column_count, "|", result.types, result.column_width);
}
const char *GetColumnSeparator() override {
return " | ";
}
const char *GetRowSeparator() override {
return " |\n";
}
const char *GetRowStart() override {
return "| ";
}
};
/*
** UTF8 box-drawing characters. Imagine box lines like this:
**
** 1
** |
** 4 --+-- 2
** |
** 3
**
** Each box characters has between 2 and 4 of the lines leading from
** the center. The characters are here identified by the numbers of
** their corresponding lines.
*/
#define BOX_24 "\342\224\200" /* U+2500 --- */
#define BOX_13 "\342\224\202" /* U+2502 | */
#define BOX_23 "\342\224\214" /* U+250c ,- */
#define BOX_34 "\342\224\220" /* U+2510 -, */
#define BOX_12 "\342\224\224" /* U+2514 '- */
#define BOX_14 "\342\224\230" /* U+2518 -' */
#define BOX_123 "\342\224\234" /* U+251c |- */
#define BOX_134 "\342\224\244" /* U+2524 -| */
#define BOX_234 "\342\224\254" /* U+252c -,- */
#define BOX_124 "\342\224\264" /* U+2534 -'- */
#define BOX_1234 "\342\224\274" /* U+253c -|- */
class ModeBoxRenderer : public ColumnRenderer {
public:
explicit ModeBoxRenderer(ShellState &state) : ColumnRenderer(state) {
}
void RenderHeader(ColumnarResult &result) override {
print_box_row_separator(result.column_count, BOX_23, BOX_234, BOX_34, result.column_width);
state.Print(BOX_13 " ");
for (idx_t i = 0; i < result.column_count; i++) {
RenderAlignedValue(result, i);
state.Print(i == result.column_count - 1 ? " " BOX_13 "\n" : " " BOX_13 " ");
}
print_box_row_separator(result.column_count, BOX_123, BOX_1234, BOX_134, result.column_width);
}
void RenderFooter(ColumnarResult &result) override {
print_box_row_separator(result.column_count, BOX_12, BOX_124, BOX_14, result.column_width);
}
const char *GetColumnSeparator() override {
return " " BOX_13 " ";
}
const char *GetRowSeparator() override {
return " " BOX_13 "\n";
}
const char *GetRowStart() override {
return BOX_13 " ";
}
private:
/* Draw horizontal line N characters long using unicode box
** characters
*/
void print_box_line(idx_t N) {
string box_line;
for (idx_t i = 0; i < N; i++) {
box_line += BOX_24;
}
state.Print(box_line);
}
/*
** Draw a horizontal separator for a RenderMode::Box table.
*/
void print_box_row_separator(int nArg, const char *zSep1, const char *zSep2, const char *zSep3,
const vector<idx_t> &actualWidth) {
int i;
if (nArg > 0) {
state.Print(zSep1);
print_box_line(actualWidth[0] + 2);
for (i = 1; i < nArg; i++) {
state.Print(zSep2);
print_box_line(actualWidth[i] + 2);
}
state.Print(zSep3);
}
state.Print("\n");
}
};
class ModeLatexRenderer : public ColumnRenderer {
public:
explicit ModeLatexRenderer(ShellState &state) : ColumnRenderer(state) {
}
void RenderHeader(ColumnarResult &result) override {
state.Print("\\begin{tabular}{|");
for (idx_t i = 0; i < result.column_count; i++) {
if (state.ColumnTypeIsInteger(result.type_names[i])) {
state.Print("r");
} else {
state.Print("l");
}
}
state.Print("|}\n");
state.Print("\\hline\n");
for (idx_t i = 0; i < result.column_count; i++) {
RenderAlignedValue(result, i);
state.Print(i == result.column_count - 1 ? GetRowSeparator() : GetColumnSeparator());
}
state.Print("\\hline\n");
}
void RenderFooter(ColumnarResult &) override {
state.Print("\\hline\n");
state.Print("\\end{tabular}\n");
}
const char *GetColumnSeparator() override {
return " & ";
}
const char *GetRowSeparator() override {
return " \\\\\n";
}
};
unique_ptr<ColumnRenderer> ShellState::GetColumnRenderer() {
switch (cMode) {
case RenderMode::COLUMN:
return unique_ptr<ColumnRenderer>(new ModeColumnRenderer(*this));
case RenderMode::TABLE:
return unique_ptr<ColumnRenderer>(new ModeTableRenderer(*this));
case RenderMode::MARKDOWN:
return unique_ptr<ColumnRenderer>(new ModeMarkdownRenderer(*this));
case RenderMode::BOX:
return unique_ptr<ColumnRenderer>(new ModeBoxRenderer(*this));
case RenderMode::LATEX:
return unique_ptr<ColumnRenderer>(new ModeLatexRenderer(*this));
default:
throw std::runtime_error("Unsupported mode for GetColumnRenderer");
}
}
//===--------------------------------------------------------------------===//
// Row Renderers
//===--------------------------------------------------------------------===//
RowRenderer::RowRenderer(ShellState &state) : ShellRenderer(state) {
}
void RowRenderer::Render(RowResult &result) {
if (first_row) {
RenderHeader(result);
first_row = false;
}
RenderRow(result);
}
void RowRenderer::RenderHeader(RowResult &result) {
}
void RowRenderer::RenderFooter(RowResult &result) {
}
class ModeLineRenderer : public RowRenderer {
public:
explicit ModeLineRenderer(ShellState &state) : RowRenderer(state) {
}
void Render(RowResult &result) override {
if (first_row) {
auto &col_names = result.column_names;
// determine the render width by going over the column names
header_width = 5;
for (idx_t i = 0; i < col_names.size(); i++) {
auto len = ShellState::StringLength(col_names[i] ? col_names[i] : "");
if (len > header_width) {
header_width = len;
}
}
first_row = false;
} else {
state.Print(state.rowSeparator);
}
// render the row
RenderRow(result);
}
void RenderRow(RowResult &result) override {
auto &data = result.data;
auto &col_names = result.column_names;
for (idx_t i = 0; i < data.size(); i++) {
state.PrintPadded(col_names[i], header_width);
state.Print(" = ");
state.PrintValue(data[i]);
state.Print(state.rowSeparator);
}
}
idx_t header_width = 0;
};
class ModeExplainRenderer : public RowRenderer {
public:
explicit ModeExplainRenderer(ShellState &state) : RowRenderer(state) {
}
void RenderRow(RowResult &result) override {
auto &data = result.data;
if (data.size() != 2) {
return;
}
if (strcmp(data[0], "logical_plan") == 0 || strcmp(data[0], "logical_opt") == 0 ||
strcmp(data[0], "physical_plan") == 0) {
state.Print("\n┌─────────────────────────────┐\n");
state.Print("│┌───────────────────────────┐│\n");
if (strcmp(data[0], "logical_plan") == 0) {
state.Print("││ Unoptimized Logical Plan ││\n");
} else if (strcmp(data[0], "logical_opt") == 0) {
state.Print("││ Optimized Logical Plan ││\n");
} else if (strcmp(data[0], "physical_plan") == 0) {
state.Print("││ Physical Plan ││\n");
}
state.Print("│└───────────────────────────┘│\n");
state.Print("└─────────────────────────────┘\n");
}
state.Print(data[1]);
}
};
class ModeListRenderer : public RowRenderer {
public:
explicit ModeListRenderer(ShellState &state) : RowRenderer(state) {
}
void RenderHeader(RowResult &result) override {
if (!show_header) {
return;
}
auto &col_names = result.column_names;
for (idx_t i = 0; i < col_names.size(); i++) {
if (i > 0) {
state.Print(col_sep);
}
state.Print(col_names[i]);
}
state.Print(row_sep);
}
void RenderRow(RowResult &result) override {
auto &data = result.data;
for (idx_t i = 0; i < data.size(); i++) {
if (i > 0) {
state.Print(col_sep);
}
state.PrintValue(data[i]);
}
state.Print(row_sep);
}
};
class ModeHtmlRenderer : public RowRenderer {
public:
explicit ModeHtmlRenderer(ShellState &state) : RowRenderer(state) {
}
void RenderHeader(RowResult &result) override {
if (!show_header) {
return;
}
auto &col_names = result.column_names;
state.Print("<tr>");
for (idx_t i = 0; i < col_names.size(); i++) {
state.Print("<th>");
output_html_string(col_names[i]);
state.Print("</th>\n");
}
state.Print("</tr>\n");
}
void RenderRow(RowResult &result) override {
auto &data = result.data;
state.Print("<tr>");
for (idx_t i = 0; i < data.size(); i++) {
state.Print("<td>");
output_html_string(data[i] ? data[i] : state.nullValue.c_str());
state.Print("</td>\n");
}
state.Print("</tr>\n");
}
/*
** Output the given string with characters that are special to
** HTML escaped.
*/
void output_html_string(const char *z) {
if (z == 0)
z = "";
string escaped;
for (; *z; z++) {
switch (*z) {
case '<':
escaped += "&lt;";
break;
case '&':
escaped += "&amp;";
break;
case '>':
escaped += "&gt;";
break;
case '\"':
escaped += "&quot;";
break;
case '\'':
escaped += "&#39;";
break;
default:
escaped += *z;
}
}
state.Print(escaped);
}
};
class ModeTclRenderer : public RowRenderer {
public:
explicit ModeTclRenderer(ShellState &state) : RowRenderer(state) {
}
void RenderHeader(RowResult &result) override {
if (!show_header) {
return;
}
auto &col_names = result.column_names;
for (idx_t i = 0; i < col_names.size(); i++) {
if (i > 0) {
state.Print(col_sep);
}
state.OutputCString(col_names[i] ? col_names[i] : "");
}
state.Print(row_sep);
}
void RenderRow(RowResult &result) override {
auto &data = result.data;
for (idx_t i = 0; i < data.size(); i++) {
if (i > 0) {
state.Print(col_sep);
}
state.OutputCString(data[i] ? data[i] : state.nullValue.c_str());
}
state.Print(row_sep);
}
};
class ModeCsvRenderer : public RowRenderer {
public:
explicit ModeCsvRenderer(ShellState &state) : RowRenderer(state) {
}
void Render(RowResult &result) override {
state.SetBinaryMode();
RowRenderer::Render(result);
state.SetTextMode();
}
void RenderHeader(RowResult &result) override {
if (!show_header) {
return;
}
auto &col_names = result.column_names;
for (idx_t i = 0; i < col_names.size(); i++) {
state.OutputCSV(col_names[i] ? col_names[i] : "", i < col_names.size() - 1);
}
state.Print(row_sep);
}
void RenderRow(RowResult &result) override {
auto &data = result.data;
for (idx_t i = 0; i < data.size(); i++) {
state.OutputCSV(data[i], i < data.size() - 1);
}
state.Print(row_sep);
}
};
class ModeAsciiRenderer : public RowRenderer {
public:
explicit ModeAsciiRenderer(ShellState &state) : RowRenderer(state) {
col_sep = "\n";
row_sep = "\n";
}
void RenderHeader(RowResult &result) override {
if (!show_header) {
return;
}
auto &col_names = result.column_names;
for (idx_t i = 0; i < col_names.size(); i++) {
if (i > 0) {
state.Print(col_sep);
}
state.Print(col_names[i] ? col_names[i] : "");
}
state.Print(row_sep);
}
void RenderRow(RowResult &result) override {
auto &data = result.data;
for (idx_t i = 0; i < data.size(); i++) {
if (i > 0) {
state.Print(col_sep);
}
state.PrintValue(data[i]);
}
state.Print(row_sep);
}
};
class ModeQuoteRenderer : public RowRenderer {
public:
explicit ModeQuoteRenderer(ShellState &state) : RowRenderer(state) {
}
void RenderHeader(RowResult &result) override {
if (!show_header) {
return;
}
auto &col_names = result.column_names;
for (idx_t i = 0; i < col_names.size(); i++) {
if (i > 0) {
state.Print(col_sep);
}
state.OutputQuotedString(col_names[i]);
}
state.Print(row_sep);
}
void RenderRow(RowResult &result) override {
auto &data = result.data;
auto &types = result.types;
for (idx_t i = 0; i < data.size(); i++) {
if (i > 0)
state.Print(col_sep);
if ((data[i] == 0) || (!types.empty() && types[i] == SQLITE_NULL)) {
state.Print("NULL");
} else if (!types.empty() && (types[i] == SQLITE_TEXT || types[i] == SQLITE_BLOB)) {
state.OutputQuotedString(data[i]);
} else if (!types.empty() && (types[i] == SQLITE_INTEGER || types[i] == SQLITE_FLOAT)) {
state.Print(data[i]);
} else if (state.IsNumber(data[i], 0)) {
state.Print(data[i]);
} else {
state.OutputQuotedString(data[i]);
}
}
state.Print(row_sep);
}
};
class ModeJsonRenderer : public RowRenderer {
public:
explicit ModeJsonRenderer(ShellState &state, bool json_array) : RowRenderer(state), json_array(json_array) {
}
void Render(RowResult &result) override {
if (first_row) {
if (json_array) {
// wrap all JSON objects in an array
state.Print("[");
}
state.Print("{");
first_row = false;
} else {
if (json_array) {
// wrap all JSON objects in an array
state.Print(",");
}
state.Print("\n{");
}
RenderRow(result);
}
void RenderRow(RowResult &result) override {
auto &data = result.data;
auto &types = result.types;
auto &col_names = result.column_names;
for (idx_t i = 0; i < col_names.size(); i++) {
if (i > 0) {
state.Print(",");
}
state.OutputJSONString(col_names[i], -1);
state.Print(":");
if ((data[i] == 0) || (!types.empty() && types[i] == SQLITE_NULL)) {
state.Print("null");
} else if (!types.empty() && types[i] == SQLITE_FLOAT) {
if (strcmp(data[i], "inf") == 0) {
state.Print("1e999");
} else if (strcmp(data[i], "-inf") == 0) {
state.Print("-1e999");
} else if (strcmp(data[i], "nan") == 0) {
state.Print("null");
} else if (strcmp(data[i], "-nan") == 0) {
state.Print("null");
} else {
state.Print(data[i]);
}
} else if (!types.empty() && types[i] == SQLITE_BLOB && result.pStmt) {
const void *pBlob = sqlite3_column_blob(result.pStmt, i);
int nBlob = sqlite3_column_bytes(result.pStmt, i);
state.OutputJSONString((const char *)pBlob, nBlob);
} else if (!types.empty() && types[i] == SQLITE_TEXT) {
state.OutputJSONString(data[i], -1);
} else {
state.Print(data[i]);
}
}
state.Print("}");
}
void RenderFooter(RowResult &result) override {
if (json_array) {
state.Print("]\n");
} else {
state.Print("\n");
}
}
bool json_array;
};
class ModeInsertRenderer : public RowRenderer {
public:
explicit ModeInsertRenderer(ShellState &state) : RowRenderer(state) {
}
void RenderRow(RowResult &result) override {
auto &data = result.data;
auto &types = result.types;
auto &col_names = result.column_names;
state.Print("INSERT INTO ");
state.Print(state.zDestTable);
if (show_header) {
state.Print("(");
for (idx_t i = 0; i < col_names.size(); i++) {
if (i > 0) {
state.Print(",");
}
state.PrintOptionallyQuotedIdentifier(col_names[i]);
}
state.Print(")");
}
for (idx_t i = 0; i < data.size(); i++) {
state.Print(i > 0 ? "," : " VALUES(");
if ((data[i] == 0) || (!types.empty() && types[i] == SQLITE_NULL)) {
state.Print("NULL");
} else if (state.IsNumber(data[i], nullptr)) {
state.Print(data[i]);
} else if (state.ShellHasFlag(SHFLG_Newlines)) {
state.OutputQuotedString(data[i]);
} else {
state.OutputQuotedEscapedString(data[i]);
}
}
state.Print(");\n");
}
};
class ModeSemiRenderer : public RowRenderer {
public:
explicit ModeSemiRenderer(ShellState &state) : RowRenderer(state) {
}
void RenderRow(RowResult &result) override {
/* .schema and .fullschema output */
state.PrintSchemaLine(result.data[0], "\n");
}
};
class ModePrettyRenderer : public RowRenderer {
public:
explicit ModePrettyRenderer(ShellState &state) : RowRenderer(state) {
}
void RenderRow(RowResult &result) override {
auto &data = result.data;
/* .schema and .fullschema with --indent */
if (data.size() != 1) {
throw std::runtime_error("row must have exactly one value for pretty rendering");
}
char *z;
int j;
int nParen = 0;
char cEnd = 0;
char c;
int nLine = 0;
if (!data[0]) {
return;
}
if (sqlite3_strlike("CREATE VIEW%", data[0], 0) == 0 || sqlite3_strlike("CREATE TRIG%", data[0], 0) == 0) {
state.Print(data[0]);
state.Print(";\n");
return;
}
z = sqlite3_mprintf("%s", data[0]);
j = 0;
idx_t i;
for (i = 0; IsSpace(z[i]); i++) {
}
for (; (c = z[i]) != 0; i++) {
if (IsSpace(c)) {
if (z[j - 1] == '\r')
z[j - 1] = '\n';
if (IsSpace(z[j - 1]) || z[j - 1] == '(')
continue;
} else if ((c == '(' || c == ')') && j > 0 && IsSpace(z[j - 1])) {
j--;
}
z[j++] = c;
}
while (j > 0 && IsSpace(z[j - 1])) {
j--;
}
z[j] = 0;
if (state.StringLength(z) >= 79) {
for (i = j = 0; (c = z[i]) != 0; i++) { /* Copy from z[i] back to z[j] */
if (c == cEnd) {
cEnd = 0;
} else if (c == '"' || c == '\'' || c == '`') {
cEnd = c;
} else if (c == '[') {
cEnd = ']';
} else if (c == '-' && z[i + 1] == '-') {
cEnd = '\n';
} else if (c == '(') {
nParen++;
} else if (c == ')') {
nParen--;
if (nLine > 0 && nParen == 0 && j > 0) {
state.PrintSchemaLineN(z, j, "\n");
j = 0;
}
}
z[j++] = c;
if (nParen == 1 && cEnd == 0 && (c == '(' || c == '\n' || (c == ',' && !wsToEol(z + i + 1)))) {
if (c == '\n')
j--;
state.PrintSchemaLineN(z, j, "\n ");
j = 0;
nLine++;
while (IsSpace(z[i + 1])) {
i++;
}
}
}
z[j] = 0;
}
state.PrintSchemaLine(z, ";\n");
sqlite3_free(z);
}
/*
** Return true if string z[] has nothing but whitespace and comments to the
** end of the first line.
*/
static bool wsToEol(const char *z) {
int i;
for (i = 0; z[i]; i++) {
if (z[i] == '\n') {
return true;
}
if (IsSpace(z[i])) {
continue;
}
if (z[i] == '-' && z[i + 1] == '-') {
return true;
}
return false;
}
return true;
}
};
unique_ptr<RowRenderer> ShellState::GetRowRenderer() {
return GetRowRenderer(cMode);
}
unique_ptr<RowRenderer> ShellState::GetRowRenderer(RenderMode mode) {
switch (mode) {
case RenderMode::LINE:
return unique_ptr<RowRenderer>(new ModeLineRenderer(*this));
case RenderMode::EXPLAIN:
return unique_ptr<RowRenderer>(new ModeExplainRenderer(*this));
case RenderMode::LIST:
return unique_ptr<RowRenderer>(new ModeListRenderer(*this));
case RenderMode::HTML:
return unique_ptr<RowRenderer>(new ModeHtmlRenderer(*this));
case RenderMode::TCL:
return unique_ptr<RowRenderer>(new ModeTclRenderer(*this));
case RenderMode::CSV:
return unique_ptr<RowRenderer>(new ModeCsvRenderer(*this));
case RenderMode::ASCII:
return unique_ptr<RowRenderer>(new ModeAsciiRenderer(*this));
case RenderMode::QUOTE:
return unique_ptr<RowRenderer>(new ModeQuoteRenderer(*this));
case RenderMode::JSON:
return unique_ptr<RowRenderer>(new ModeJsonRenderer(*this, true));
case RenderMode::JSONLINES:
return unique_ptr<RowRenderer>(new ModeJsonRenderer(*this, false));
case RenderMode::INSERT:
return unique_ptr<RowRenderer>(new ModeInsertRenderer(*this));
case RenderMode::SEMI:
return unique_ptr<RowRenderer>(new ModeSemiRenderer(*this));
case RenderMode::PRETTY:
return unique_ptr<RowRenderer>(new ModePrettyRenderer(*this));
default:
throw std::runtime_error("Unsupported mode for GetRowRenderer");
}
}
} // namespace duckdb_shell

180
external/duckdb/tools/shell/tests/conftest.py vendored Normal file
View File

@@ -0,0 +1,180 @@
import pytest
import os
import subprocess
import sys
from typing import List, NamedTuple, Union
def pytest_addoption(parser):
parser.addoption(
"--shell-binary", action="store", default=None, help="Provide the shell binary to use for the tests"
)
parser.addoption("--start-offset", action="store", type=int, help="Skip the first 'n' tests")
def pytest_collection_modifyitems(config, items):
start_offset = config.getoption("--start-offset")
if not start_offset:
# --skiplist not given in cli, therefore move on
return
skipped = pytest.mark.skip(reason="included in --skiplist")
skipped_items = items[:start_offset]
for item in skipped_items:
item.add_marker(skipped)
class TestResult:
def __init__(self, stdout, stderr, status_code):
self.stdout: Union[str, bytes] = stdout
self.stderr: Union[str, bytes] = stderr
self.status_code: int = status_code
def check_stdout(self, expected: Union[str, List[str], bytes]):
if isinstance(expected, list):
expected = '\n'.join(expected)
assert self.status_code == 0
assert expected in self.stdout
def check_not_exist(self, not_exist: Union[str, List[str], bytes]):
if isinstance(not_exist, list):
not_exist = '\n'.join(not_exist)
assert self.status_code == 0
assert not_exist not in self.stdout
def check_stderr(self, expected: str):
assert expected in self.stderr
class ShellTest:
def __init__(self, shell, arguments=[]):
if not shell:
raise ValueError("Please provide a shell binary")
self.shell = shell
self.arguments = [shell, '--batch', '--init', '/dev/null'] + arguments
self.statements: List[str] = []
self.input = None
self.output = None
self.environment = {}
def add_argument(self, *args):
self.arguments.extend(args)
return self
def statement(self, stmt):
self.statements.append(stmt)
return self
def query(self, *stmts):
self.statements.extend(stmts)
return self
def input_file(self, file_path):
self.input = file_path
return self
def output_file(self, file_path):
self.output = file_path
return self
# Test Running methods
def get_command(self, cmd: str) -> List[str]:
command = self.arguments
if self.input:
command += [cmd]
return command
def get_input_data(self, cmd: str):
if self.input:
input_data = open(self.input, 'rb').read()
else:
input_data = bytearray(cmd, 'utf8')
return input_data
def get_output_pipe(self):
output_pipe = subprocess.PIPE
if self.output:
output_pipe = open(self.output, 'w+')
return output_pipe
def get_statements(self):
result = ""
statements = []
for statement in self.statements:
if statement.startswith('.'):
statements.append(statement)
else:
statements.append(statement + ';')
return '\n'.join(statements)
def get_output_data(self, res):
if self.output:
stdout = open(self.output, 'r').read()
else:
stdout = res.stdout.decode('utf8').strip()
stderr = res.stderr.decode('utf8').strip()
return stdout, stderr
def run(self):
statements = self.get_statements()
command = self.get_command(statements)
input_data = self.get_input_data(statements)
output_pipe = self.get_output_pipe()
my_env = os.environ.copy()
for key, val in self.environment.items():
my_env[key] = val
res = subprocess.run(command, input=input_data, stdout=output_pipe, stderr=subprocess.PIPE, env=my_env)
stdout, stderr = self.get_output_data(res)
return TestResult(stdout, stderr, res.returncode)
@pytest.fixture()
def shell(request):
custom_arg = request.config.getoption("--shell-binary")
if not custom_arg:
raise ValueError("Please provide a shell binary path to the tester, using '--shell-binary <path_to_cli>'")
return custom_arg
@pytest.fixture()
def random_filepath(request, tmp_path):
tmp_file = tmp_path / "random_import_file"
return tmp_file
@pytest.fixture()
def generated_file(request, random_filepath):
param = request.param
tmp_file = random_filepath
with open(tmp_file, 'w+') as f:
f.write(param)
return tmp_file
def check_load_status(shell, extension: str):
binary = ShellTest(shell)
binary.statement(f"select loaded from duckdb_extensions() where extension_name = '{extension}';")
result = binary.run()
return result.stdout
def assert_loaded(shell, extension: str):
# TODO: add a command line argument to fail instead of skip if the extension is not loaded
out = check_load_status(shell, extension)
if 'true' not in out:
pytest.skip(reason=f"'{extension}' extension is not loaded!")
return
@pytest.fixture()
def autocomplete_extension(shell):
assert_loaded(shell, 'autocomplete')
@pytest.fixture()
def json_extension(shell):
assert_loaded(shell, 'json')

340
external/duckdb/tools/shell/tests/test_autocomplete.py vendored Normal file
View File

@@ -0,0 +1,340 @@
# fmt: off
import pytest
import subprocess
import sys
from typing import List
from conftest import ShellTest
from conftest import autocomplete_extension
import os
# 'autocomplete_extension' is a fixture which will skip the test if 'autocomplete' is not loaded
def test_autocomplete_select(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CALL sql_auto_complete('SEL')")
)
result = test.run()
result.check_stdout('SELECT')
def test_autocomplete_first_from(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CALL sql_auto_complete('FRO')")
)
result = test.run()
result.check_stdout('FROM')
def test_autocomplete_column(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CREATE TABLE my_table(my_column INTEGER);")
.statement("SELECT * FROM sql_auto_complete('SELECT my_') LIMIT 1;")
)
result = test.run()
result.check_stdout('my_column')
def test_autocomplete_where(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CREATE TABLE my_table(my_column INTEGER);")
.statement("SELECT * FROM sql_auto_complete('SELECT my_column FROM my_table WH') LIMIT 1;")
)
result = test.run()
result.check_stdout('WHERE')
def test_autocomplete_insert(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CREATE TABLE my_table(my_column INTEGER);")
.statement("SELECT * FROM sql_auto_complete('INS') LIMIT 1;")
)
result = test.run()
result.check_stdout('INSERT')
def test_autocomplete_into(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CREATE TABLE my_table(my_column INTEGER);")
.statement("SELECT * FROM sql_auto_complete('INSERT IN') LIMIT 1;")
)
result = test.run()
result.check_stdout('INTO')
def test_autocomplete_into_table(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CREATE TABLE my_table(my_column INTEGER);")
.statement("SELECT * FROM sql_auto_complete('INSERT INTO my_t') LIMIT 1;")
)
result = test.run()
result.check_stdout('my_table')
def test_autocomplete_values(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CREATE TABLE my_table(my_column INTEGER);")
.statement("SELECT * FROM sql_auto_complete('INSERT INTO my_table VAL') LIMIT 1;")
)
result = test.run()
result.check_stdout('VALUES')
def test_autocomplete_delete(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CREATE TABLE my_table(my_column INTEGER);")
.statement("SELECT * FROM sql_auto_complete('DEL') LIMIT 1;")
)
result = test.run()
result.check_stdout('DELETE')
def test_autocomplete_delete_from(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CREATE TABLE my_table(my_column INTEGER);")
.statement("SELECT * FROM sql_auto_complete('DELETE F') LIMIT 1;")
)
result = test.run()
result.check_stdout('FROM')
def test_autocomplete_from_table(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CREATE TABLE my_table(my_column INTEGER);")
.statement("SELECT * FROM sql_auto_complete('DELETE FROM m') LIMIT 1;")
)
result = test.run()
result.check_stdout('my_table')
def test_autocomplete_update(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CREATE TABLE my_table(my_column INTEGER);")
.statement("SELECT * FROM sql_auto_complete('UP') LIMIT 1;")
)
result = test.run()
result.check_stdout('UPDATE')
def test_autocomplete_update_table(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CREATE TABLE my_table(my_column INTEGER);")
.statement("SELECT * FROM sql_auto_complete('UPDATE m') LIMIT 1;")
)
result = test.run()
result.check_stdout('my_table')
test = (
ShellTest(shell)
.statement("CREATE TABLE my_table(my_column INTEGER);")
.statement("""SELECT * FROM sql_auto_complete('UPDATE "m') LIMIT 1;""")
)
result = test.run()
result.check_stdout('my_table')
def test_autocomplete_update_column(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CREATE TABLE my_table(my_column INTEGER);")
.statement("SELECT * FROM sql_auto_complete('UPDATE my_table SET m') LIMIT 1;")
)
result = test.run()
result.check_stdout('my_column')
def test_autocomplete_funky_table(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("""CREATE TABLE "Funky Table With Spaces"(my_column INTEGER);""")
.statement("SELECT * FROM sql_auto_complete('SELECT * FROM F') LIMIT 1;")
)
result = test.run()
result.check_stdout('"Funky Table With Spaces"')
test = (
ShellTest(shell)
.statement("""CREATE TABLE "Funky Table With Spaces"("Funky Column" int);""")
.statement("""SELECT * FROM sql_auto_complete('select "Funky Column" FROM f') LIMIT 1;""")
)
result = test.run()
result.check_stdout('"Funky Table With Spaces"')
def test_autocomplete_funky_column(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("""CREATE TABLE "Funky Table With Spaces"("Funky Column" int);""")
.statement("SELECT * FROM sql_auto_complete('select f') LIMIT 1;")
)
result = test.run()
result.check_stdout('"Funky Column"')
def test_autocomplete_semicolon(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("SELECT * FROM sql_auto_complete('SELECT 42; SEL') LIMIT 1;")
)
result = test.run()
result.check_stdout('SELECT')
def test_autocomplete_comments(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("""
SELECT * FROM sql_auto_complete('--SELECT * FROM
SEL') LIMIT 1;""")
)
result = test.run()
result.check_stdout('SELECT')
def test_autocomplete_scalar_functions(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("SELECT * FROM sql_auto_complete('SELECT regexp_m') LIMIT 1;")
)
result = test.run()
result.check_stdout('regexp_matches')
def test_autocomplete_aggregates(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("SELECT * FROM sql_auto_complete('SELECT approx_c') LIMIT 1;")
)
result = test.run()
result.check_stdout('approx_count_distinct')
def test_autocomplete_builtin_views(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("SELECT * FROM sql_auto_complete('SELECT * FROM sqlite_ma') LIMIT 1;")
)
result = test.run()
result.check_stdout('sqlite_master')
def test_autocomplete_table_function(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("SELECT * FROM sql_auto_complete('SELECT * FROM read_csv_a') LIMIT 1;")
)
result = test.run()
result.check_stdout('read_csv_auto')
def test_autocomplete_tpch(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CREATE TABLE partsupp(ps_suppkey int);")
.statement("CREATE TABLE supplier(s_suppkey int);")
.statement("CREATE TABLE nation(n_nationkey int);")
.statement("SELECT * FROM sql_auto_complete('DROP TABLE na') LIMIT 1;")
)
result = test.run()
result.check_stdout('nation')
test = (
ShellTest(shell)
.statement("CREATE TABLE partsupp(ps_suppkey int);")
.statement("CREATE TABLE supplier(s_suppkey int);")
.statement("CREATE TABLE nation(n_nationkey int);")
.statement("SELECT * FROM sql_auto_complete('SELECT s_supp') LIMIT 1;")
)
result = test.run()
result.check_stdout('s_suppkey')
test = (
ShellTest(shell)
.statement("CREATE TABLE partsupp(ps_suppkey int);")
.statement("CREATE TABLE supplier(s_suppkey int);")
.statement("CREATE TABLE nation(n_nationkey int);")
.statement("SELECT * FROM sql_auto_complete('SELECT * FROM partsupp JOIN supp') LIMIT 1;")
)
result = test.run()
result.check_stdout('supplier')
test = (
ShellTest(shell)
.statement("CREATE TABLE partsupp(ps_suppkey int);")
.statement("CREATE TABLE supplier(s_suppkey int);")
.statement("CREATE TABLE nation(n_nationkey int);")
.statement(".mode csv")
.statement("SELECT l,l FROM sql_auto_complete('SELECT * FROM partsupp JOIN supplier ON (s_supp') t(l) LIMIT 1;")
)
result = test.run()
result.check_stdout('s_suppkey,s_suppkey')
test = (
ShellTest(shell)
.statement("CREATE TABLE partsupp(ps_suppkey int);")
.statement("CREATE TABLE supplier(s_suppkey int);")
.statement("CREATE TABLE nation(n_nationkey int);")
.statement("SELECT * FROM sql_auto_complete('SELECT * FROM partsupp JOIN supplier USING (ps_su') LIMIT 1;")
)
result = test.run()
result.check_stdout('ps_suppkey')
def test_autocomplete_from(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("SELECT * FROM sql_auto_complete('SELECT * FR') LIMIT 1;")
)
result = test.run()
result.check_stdout('FROM')
def test_autocomplete_disambiguation_column(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CREATE TABLE MyTable(MyColumn Varchar);")
.statement("SELECT * FROM sql_auto_complete('SELECT My') LIMIT 1;")
)
result = test.run()
result.check_stdout('MyColumn')
def test_autocomplete_disambiguation_table(shell, autocomplete_extension):
test = (
ShellTest(shell)
.statement("CREATE TABLE MyTable(MyColumn Varchar);")
.statement("SELECT * FROM sql_auto_complete('SELECT MyColumn FROM My') LIMIT 1;")
)
result = test.run()
result.check_stdout('MyTable')
def test_autocomplete_directory(shell, autocomplete_extension, tmp_path):
shell_test_dir = tmp_path / 'shell_test_dir'
extra_path = tmp_path / 'shell_test_dir' / 'extra_path'
shell_test_dir.mkdir()
extra_path.mkdir()
# Create the files
base_files = ['extra.parquet', 'extra.file']
for fname in base_files:
with open(shell_test_dir / fname, 'w+') as f:
f.write('')
# Complete the directory
partial_directory = tmp_path / 'shell_test'
test = (
ShellTest(shell)
.statement("CREATE TABLE MyTable(MyColumn Varchar);")
.statement(f"SELECT * FROM sql_auto_complete('SELECT * FROM ''{partial_directory.as_posix()}') LIMIT 1;")
)
result = test.run()
result.check_stdout("shell_test_dir")
# Complete the sub directory as well
partial_subdirectory = tmp_path / 'shell_test_dir' / 'extra'
test = (
ShellTest(shell)
.statement("CREATE TABLE MyTable(MyColumn Varchar);")
.statement(f"SELECT * FROM sql_auto_complete('SELECT * FROM ''{partial_subdirectory.as_posix()}') LIMIT 1;")
)
result = test.run()
result.check_stdout("extra_path")
# Complete the parquet file in the sub directory
partial_parquet = tmp_path / 'shell_test_dir' / 'extra.par'
test = (
ShellTest(shell)
.statement("CREATE TABLE MyTable(MyColumn Varchar);")
.statement(f"SELECT * FROM sql_auto_complete('SELECT * FROM ''{partial_parquet.as_posix()}') LIMIT 1;")
)
result = test.run()
result.check_stdout("extra.parquet")
# fmt: on

58
external/duckdb/tools/shell/tests/test_backwards_compatibility.py vendored Normal file
View File

@@ -0,0 +1,58 @@
# fmt: off
import pytest
import subprocess
import sys
from typing import List
from conftest import ShellTest
import os
def test_version_dev(shell):
test = (
ShellTest(shell)
.statement(".open test/storage/bc/db_dev.db")
)
result = test.run()
result.check_stderr("older development version")
def test_version_0_3_1(shell):
test = (
ShellTest(shell)
.statement(".open test/storage/bc/db_031.db")
)
result = test.run()
result.check_stderr("v0.3.1")
def test_version_0_3_2(shell):
test = (
ShellTest(shell)
.statement(".open test/storage/bc/db_032.db")
)
result = test.run()
result.check_stderr("v0.3.2")
def test_version_0_4(shell):
test = (
ShellTest(shell)
.statement(".open test/storage/bc/db_04.db")
)
result = test.run()
result.check_stderr("v0.4.0")
def test_version_0_5_1(shell):
test = (
ShellTest(shell)
.statement(".open test/storage/bc/db_051.db")
)
result = test.run()
result.check_stderr("v0.5.1")
def test_version_0_6_0(shell):
test = (
ShellTest(shell)
.statement(".open test/storage/bc/db_060.db")
)
result = test.run()
result.check_stderr("v0.6.0")
# fmt: on

95
external/duckdb/tools/shell/tests/test_errors.py vendored Normal file
View File

@@ -0,0 +1,95 @@
# fmt: off
import pytest
import subprocess
import sys
from typing import List
from conftest import ShellTest
import os
lineitem_ddl = 'CREATE TABLE lineitem(l_orderkey BIGINT NOT NULL, l_partkey BIGINT NOT NULL, l_suppkey BIGINT NOT NULL, l_linenumber BIGINT NOT NULL, l_quantity DECIMAL(15,2) NOT NULL, l_extendedprice DECIMAL(15,2) NOT NULL, l_discount DECIMAL(15,2) NOT NULL, l_tax DECIMAL(15,2) NOT NULL, l_returnflag VARCHAR NOT NULL, l_linestatus VARCHAR NOT NULL, l_shipdate DATE NOT NULL, l_commitdate DATE NOT NULL, l_receiptdate DATE NOT NULL, l_shipinstruct VARCHAR NOT NULL, l_shipmode VARCHAR NOT NULL, l_comment VARCHAR NOT NULL);'
@pytest.mark.skipif(os.name == 'nt', reason="Windows highlighting does not use shell escapes")
def test_incorrect_column(shell):
test = (
ShellTest(shell)
.statement(".highlight_errors on")
.statement(lineitem_ddl)
.statement('select * from lineitem where l_extendedpric=5;')
)
result = test.run()
result.check_stderr('"\x1b[33ml_extendedprice')
result.check_stderr('"\x1b[33ml_extendedpric\x1b[0m')
@pytest.mark.skipif(os.name == 'nt', reason="Windows highlighting does not use shell escapes")
def test_missing_table(shell):
test = (
ShellTest(shell)
.statement(".highlight_errors on")
.statement(lineitem_ddl)
.statement('select * from lineite where l_extendedprice=5;')
)
result = test.run()
result.check_stderr('"\x1b[33mlineitem\x1b[0m')
@pytest.mark.skipif(os.name == 'nt', reason="Windows highlighting does not use shell escapes")
def test_long_error(shell):
test = (
ShellTest(shell)
.statement(".highlight_errors on")
.statement(lineitem_ddl)
.statement('''SELECT
l_returnflag,
l_linestatus,
sum(l_quantity) AS sum_qty,
sum(l_extendedprice) AS sum_base_price,
sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price,
sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge,
avg(l_quantity) AS avg_qty,
avg(l_extendedprice) AS avg_price,
avg(l_discount) AS avg_disc,
count(*) AS count_order
FROM
lineitem
WHERE
l_shipdate <= CAST('1998-09-02' AS date) + timestamp '2020-01-01'
GROUP BY
l_returnflag,
l_linestatus
ORDER BY
l_returnflag,
l_linestatus;''')
)
result = test.run()
result.check_stderr('\x1b[33m+(DATE, TIMESTAMP)\x1b[0m')
result.check_stderr('\x1b[32mCAST\x1b[0m')
@pytest.mark.skipif(os.name == 'nt', reason="Windows highlighting does not use shell escapes")
def test_single_quotes_in_error(shell):
test = (
ShellTest(shell)
.statement(".highlight_errors on")
.statement("select \"I'm an error\"")
)
result = test.run()
result.check_stderr('"\x1b[33mI\'m an error\x1b[0m')
@pytest.mark.skipif(os.name == 'nt', reason="Windows highlighting does not use shell escapes")
def test_double_quotes_in_error(shell):
test = (
ShellTest(shell)
.statement(".highlight_errors on")
.statement("select error('''I\"m an error''')")
)
result = test.run()
result.check_stderr('\x1b[33mI"m an error\x1b[0m')
@pytest.mark.skipif(os.name == 'nt', reason="Windows highlighting does not use shell escapes")
def test_unterminated_quote(shell):
test = (
ShellTest(shell)
.statement(".highlight_errors on")
.statement("select error('I''m an error')")
)
result = test.run()
result.check_stderr('I\'m an error')

15
external/duckdb/tools/shell/tests/test_explain.py vendored Normal file
View File

@@ -0,0 +1,15 @@
# fmt: off
import pytest
import subprocess
import sys
from typing import List
from conftest import ShellTest
import os
def test_invalid_explain(shell):
test = (
ShellTest(shell)
.statement("EXPLAIN SELECT 'any_string' IN ?;")
)
result = test.run()

30
external/duckdb/tools/shell/tests/test_get_env.py vendored Normal file
View File

@@ -0,0 +1,30 @@
# fmt: off
from conftest import ShellTest
def test_get_env(shell):
test = (
ShellTest(shell)
.statement('.null NULL')
.statement("SET default_null_order=getenv('DEFAULT_NULL_ORDER');")
.statement("SELECT * FROM (VALUES (42), (NULL)) ORDER BY 1 LIMIT 1;")
)
test.environment['DEFAULT_NULL_ORDER'] = 'NULLS_FIRST'
result = test.run()
result.check_stdout('NULL')
test.environment['DEFAULT_NULL_ORDER'] = 'NULLS_LAST'
result = test.run()
result.check_stdout('42')
def test_get_env_permissions(shell):
test = (
ShellTest(shell)
.statement('SET enable_external_access=false')
.statement("SELECT getenv('DEFAULT_NULL_ORDER');")
)
test.environment['DEFAULT_NULL_ORDER'] = 'NULLS_FIRST'
result = test.run()
result.check_stderr('disabled through configuration')
# fmt: on

49
external/duckdb/tools/shell/tests/test_highlighting.py vendored Normal file
View File

@@ -0,0 +1,49 @@
# fmt: off
import pytest
import subprocess
import sys
from typing import List
from conftest import ShellTest
import os
lineitem_ddl = 'CREATE TABLE lineitem(l_orderkey BIGINT NOT NULL, l_partkey BIGINT NOT NULL, l_suppkey BIGINT NOT NULL, l_linenumber BIGINT NOT NULL, l_quantity DECIMAL(15,2) NOT NULL, l_extendedprice DECIMAL(15,2) NOT NULL, l_discount DECIMAL(15,2) NOT NULL, l_tax DECIMAL(15,2) NOT NULL, l_returnflag VARCHAR NOT NULL, l_linestatus VARCHAR NOT NULL, l_shipdate DATE NOT NULL, l_commitdate DATE NOT NULL, l_receiptdate DATE NOT NULL, l_shipinstruct VARCHAR NOT NULL, l_shipmode VARCHAR NOT NULL, l_comment VARCHAR NOT NULL);'
@pytest.mark.skipif(os.name == 'nt', reason="Windows highlighting does not use shell escapes")
def test_highlight_column_header(shell):
test = (
ShellTest(shell)
.statement(".highlight_results on")
.statement('select NULL AS r;')
)
result = test.run()
result.check_stdout('\x1b[90mNULL\x1b[0m')
@pytest.mark.skipif(os.name == 'nt', reason="Windows highlighting does not use shell escapes")
def test_custom_highlight(shell):
test = (
ShellTest(shell)
.statement(".highlight_results on")
.statement(".highlight_colors column_name red bold")
.statement(".highlight_colors column_type yellow")
.statement(lineitem_ddl)
.statement('select * from lineitem;')
)
result = test.run()
result.check_stdout('\x1b[1m\x1b[31ml_comment\x1b[0m')
result.check_stdout('\x1b[33mvarchar\x1b[0m')
def test_custom_highlight_error(shell):
test = (
ShellTest(shell)
.statement(".highlight_colors column_nameXX red")
.statement(".highlight_colors column_name redXX")
.statement(".highlight_colors column_name red boldXX")
.statement(".highlight_colors column_name red bold zz")
)
result = test.run()
result.check_stderr("Unknown element 'column_nameXX'")
result.check_stderr("Unknown color 'redXX'")
result.check_stderr("Unknown intensity 'boldXX'")
result.check_stderr("Usage: .highlight_colors")
# fmt: on

27
external/duckdb/tools/shell/tests/test_http_logging.py vendored Normal file
View File

@@ -0,0 +1,27 @@
# fmt: off
import pytest
import subprocess
import sys
from typing import List
from conftest import ShellTest
import os
@pytest.mark.skip(reason="Skip after File Logging rework")
def test_http_logging_file(shell, tmp_path):
temp_dir = tmp_path / 'http_logging_dir'
temp_dir.mkdir()
temp_file = temp_dir / 'myfile'
test = (
ShellTest(shell)
.statement("SET enable_http_logging=true;")
.statement(f"SET http_logging_output='{temp_file.as_posix()}'")
.statement("install 'http://extensions.duckdb.org/v0.10.1/osx_arm64/httpfs.duckdb_extension.gzzz';")
)
result = test.run()
with open(temp_file, 'r') as f:
file_content = f.read()
assert "HTTP Request" in file_content
assert "HTTP Response" in file_content

29
external/duckdb/tools/shell/tests/test_import_export_database.py vendored Normal file
View File

@@ -0,0 +1,29 @@
# fmt: off
import pytest
import subprocess
import sys
from typing import List
from conftest import ShellTest
def test_profiling_json(shell, tmp_path):
target_dir = tmp_path / 'export_test'
test = (
ShellTest(shell)
.statement(".mode csv")
.statement(".changes off")
.statement("CREATE TABLE integers(i INTEGER);")
.statement("CREATE TABLE integers2(i INTEGER);")
.statement("INSERT INTO integers SELECT * FROM range(100);")
.statement("INSERT INTO integers2 VALUES (1), (3), (99);")
.statement(f"EXPORT DATABASE '{target_dir.as_posix()}';")
.statement("DROP TABLE integers;")
.statement("DROP TABLE integers2;")
.statement(f"IMPORT DATABASE '{target_dir.as_posix()}';")
.statement("SELECT SUM(i)*MAX(i) FROM integers JOIN integers2 USING (i);")
)
result = test.run()
result.check_stdout('10197')
# fmt: on

43
external/duckdb/tools/shell/tests/test_logging.py vendored Normal file
View File

@@ -0,0 +1,43 @@
# fmt: off
import pytest
import subprocess
import sys
import os
from typing import List
from conftest import ShellTest
def test_logging(shell):
test = (
ShellTest(shell)
.statement("CALL enable_logging('QueryLog', storage='stdout')")
.statement('SELECT 1 as a')
)
result = test.run()
newline = "\r\n" if os.name == "nt" else "\n"
result.check_stdout(f"QueryLog\tINFO\tSELECT 1 as a;{newline}┌───────┐")
def test_logging_custom_delim(shell):
test = (
ShellTest(shell)
.statement("CALL enable_logging('QueryLog', storage='stdout', storage_config={'delim':','})")
.statement('SELECT 1 as a')
)
result = test.run()
newline = "\r\n" if os.name == "nt" else "\n"
result.check_stdout(f"QueryLog,INFO,SELECT 1 as a;{newline}┌───────┐")
# By default stdoutlogging has buffer size of 1, but we can increase it if we want. We use `only_flush_on_full_buffer` to ensure we can test this
def test_logging_buffering(shell):
test = (
ShellTest(shell)
.statement("CALL enable_logging('QueryLog', storage='stdout', storage_buffer_size=1000, storage_config={'only_flush_on_full_buffer': true})")
.statement('SELECT 1 as a')
.statement('SELECT 2 as b')
)
result = test.run()
result.check_not_exist("QueryLog")
# fmt: on

32
external/duckdb/tools/shell/tests/test_meta_transaction.py vendored Normal file
View File

@@ -0,0 +1,32 @@
# fmt: off
from conftest import ShellTest
def test_temp_directory(shell):
test = (
ShellTest(shell)
.statement(".mode csv")
.statement("CREATE SEQUENCE id_seq;")
.statement("""
CREATE TABLE my_table (
id INTEGER DEFAULT nextval('id_seq'),
a INTEGER
);""")
.statement("ATTACH ':memory:' AS s1;")
.statement("CREATE TABLE s1.tbl AS FROM range(2000000);")
.statement("INSERT INTO my_table (a) SELECT * FROM s1.tbl;")
.statement("INSERT INTO my_table (a) SELECT * FROM s1.tbl;")
.statement("INSERT INTO my_table (a) SELECT * FROM s1.tbl;")
.statement("INSERT INTO my_table (a) SELECT * FROM s1.tbl;")
.statement("INSERT INTO my_table (a) SELECT * FROM s1.tbl;")
.statement("INSERT INTO my_table (a) SELECT * FROM s1.tbl;")
.statement("INSERT INTO my_table (a) SELECT * FROM s1.tbl;")
.statement("INSERT INTO my_table (a) SELECT * FROM s1.tbl;")
.statement("INSERT INTO my_table (a) SELECT * FROM s1.tbl;")
.statement("INSERT INTO my_table (a) SELECT * FROM s1.tbl;")
)
test = test.statement("select count(*) from my_table")
result = test.run()
result.check_stdout("20000000")
# fmt: on

23
external/duckdb/tools/shell/tests/test_profiling.py vendored Normal file
View File

@@ -0,0 +1,23 @@
# fmt: off
import pytest
import subprocess
import sys
from typing import List
from conftest import ShellTest
def test_profiling_json(shell):
test = (
ShellTest(shell)
.statement("PRAGMA enable_profiling=json;")
.statement('CREATE TABLE "foo"("hello world" INT);')
.statement("""SELECT "hello world", '\r\t\n\b\f\\' FROM "foo";""")
)
result = test.run()
result.check_stderr(r'"hello world"')
# This is incorrectly split but that's impossible to do correctly currently.
result.check_stderr(r''''\r\t"''')
result.check_stderr(r""""\b\f\\'""")
# fmt: on

278
external/duckdb/tools/shell/tests/test_read_from_stdin.py vendored Normal file
View File

@@ -0,0 +1,278 @@
# fmt: off
import pytest
import subprocess
import sys
from typing import List
from conftest import ShellTest
from conftest import json_extension
import os
@pytest.mark.skipif(os.name == 'nt', reason="Skipped on windows")
class TestReadFromStdin(object):
def test_read_stdin_csv(self, shell):
test = (
ShellTest(shell)
.input_file('data/csv/test/test.csv')
.statement("""
create table mytable as select * from
read_csv('/dev/stdin',
columns=STRUCT_PACK(foo := 'INTEGER', bar := 'INTEGER', baz := 'VARCHAR'),
AUTO_DETECT='false'
)
""")
.statement("select * from mytable limit 1;")
.add_argument(
'-csv',
':memory:'
)
)
result = test.run()
result.check_stdout("foo,bar,baz")
result.check_stdout('0,0, test')
def test_read_stdin_csv_where_filename(self, shell):
test = (
ShellTest(shell)
.input_file('data/csv/test/test.csv')
.statement("""
SELECT * FROM read_csv_auto(
'data/csv/bug_9005/teste*.csv',
header=TRUE,
filename=true,
union_by_name=True
) where filename='data/csv/bug_9005/teste1.csv'
""")
.add_argument(
'-csv',
':memory:'
)
)
result = test.run()
result.check_stdout([
'id,name,factor,filename',
'1,Ricardo,1.5,data/csv/bug_9005/teste1.csv',
'2,Jose,2.0,data/csv/bug_9005/teste1.csv'
])
def test_read_stdin_csv_auto(self, shell):
test = (
ShellTest(shell)
.input_file('data/csv/test/test.csv')
.statement("""
create table mytable as select * from
read_csv_auto('/dev/stdin')
""")
.statement("select * from mytable limit 1;")
.add_argument(
'-csv',
':memory:'
)
)
result = test.run()
result.check_stdout("column0,column1,column2")
result.check_stdout('0,0, test')
def test_split_part_csv(self, shell):
test = (
ShellTest(shell)
.input_file('data/csv/split_part.csv')
.statement("""
FROM read_csv('/dev/stdin') select split_part(C1, ',', 2) as res;
""")
.add_argument(
'-csv',
':memory:'
)
)
result = test.run()
result.check_stdout("res")
result.check_stdout('12')
result.check_stdout('12')
def test_read_stdin_csv_auto_projection(self, shell):
test = (
ShellTest(shell)
.input_file('data/csv/tpcds_14.csv')
.statement("""
create table mytable as select * from
read_csv_auto('/dev/stdin')
""")
.statement("select channel,i_brand_id,sum_sales,number_sales from mytable;")
.add_argument(
'-csv',
':memory:'
)
)
result = test.run()
result.check_stdout("web,8006004,844.21,21")
def test_read_stdin_ndjson(self, shell, json_extension):
test = (
ShellTest(shell)
.input_file('data/json/example_rn.ndjson')
.statement("""
create table mytable as select * from
read_ndjson_objects('/dev/stdin')
""")
.statement("select * from mytable;")
.add_argument(
'-list',
':memory:'
)
)
result = test.run()
result.check_stdout([
"json",
'{"id":1,"name":"O Brother, Where Art Thou?"}',
'{"id":2,"name":"Home for the Holidays"}',
'{"id":3,"name":"The Firm"}',
'{"id":4,"name":"Broadcast News"}',
'{"id":5,"name":"Raising Arizona"}'
])
def test_read_stdin_json_auto(self, shell, json_extension):
test = (
ShellTest(shell)
.input_file('data/json/example_rn.ndjson')
.statement("""
create table mytable as select * from
read_json_auto('/dev/stdin')
""")
.statement("select * from mytable;")
.add_argument(
'-list',
':memory:'
)
)
result = test.run()
result.check_stdout([
'id|name',
'1|O Brother, Where Art Thou?',
'2|Home for the Holidays',
'3|The Firm',
'4|Broadcast News',
'5|Raising Arizona'
])
def test_read_stdin_json_array(self, shell, json_extension):
test = (
ShellTest(shell)
.input_file('data/json/11407.json')
.statement("""
create table mytable as select * from
read_json_auto('/dev/stdin')
""")
.statement("select * from mytable;")
.add_argument(
'-list',
':memory:'
)
)
result = test.run()
result.check_stdout([
'k',
'v',
'v2'
])
def test_read_stdin_json_auto_recursive_cte(self, shell, json_extension):
# FIXME: disabled for now
return
test = (
ShellTest(shell)
.input_file('data/json/filter_keystage.ndjson')
.statement("""
CREATE TABLE mytable AS
WITH RECURSIVE nums AS (
SELECT 0 AS n
UNION ALL
SELECT n + 1
FROM nums
WHERE n < (
SELECT MAX(JSON_ARRAY_LENGTH(filter_keystage))::int - 1
FROM read_json_auto('/dev/stdin'))
) SELECT * FROM nums;
""")
.statement("select * from mytable;")
.add_argument(
'-list',
':memory:'
)
)
result = test.run()
result.check_stdout([
'n',
'0',
'1',
'2',
])
@pytest.mark.parametrize("alias", [
"'/dev/stdout'",
'stdout'
])
def test_copy_to_stdout(self, shell, alias):
test = (
ShellTest(shell)
.statement(f"COPY (SELECT 42) TO {alias};")
)
result = test.run()
result.check_stdout('42')
@pytest.mark.parametrize("alias", [
"'/dev/stdout'",
'stdout'
])
def test_copy_csv_to_stdout(self, shell, alias):
test = (
ShellTest(shell)
.statement(f"COPY (SELECT 42) TO {alias} WITH (FORMAT 'csv');")
.add_argument(
'-csv',
':memory:'
)
)
result = test.run()
result.check_stdout('42')
@pytest.mark.parametrize("alias", [
"'/dev/stderr'"
])
def test_copy_csv_to_stderr(self, shell, alias):
test = (
ShellTest(shell)
.statement(f"COPY (SELECT 42) TO {alias} WITH (FORMAT 'csv');")
.add_argument(
'-csv',
':memory:'
)
)
result = test.run()
result.check_stderr('42')
def test_copy_non_inlined_string(self, shell):
test = (
ShellTest(shell)
.statement("select list(concat('thisisalongstring', range::VARCHAR)) i from range(10000)")
)
result = test.run()
result.check_stdout('thisisalongstring')
def test_write_to_stdout_piped_to_file(self, shell, random_filepath):
test = (
ShellTest(shell)
.statement("copy (select * from range(10000) tbl(i)) to '/dev/stdout' (format csv)")
.output_file(random_filepath.as_posix())
)
result = test.run()
result.check_stdout('9999')
# fmt: on

151
external/duckdb/tools/shell/tests/test_readable_numbers.py vendored Normal file
View File

@@ -0,0 +1,151 @@
# fmt: off
import pytest
import subprocess
import sys
from typing import List
from conftest import ShellTest
import os
def test_readable_numbers(shell):
test = (
ShellTest(shell)
.statement(".large_number_rendering footer")
.statement("select 59986052 as count, 123456789 as count2, 9999999999 count3, -9999999999 count4;")
)
result = test.run()
result.check_stdout("(59.99 million)")
result.check_stdout("(123.46 million)")
result.check_stdout("(10.00 billion)")
result.check_stdout("(-10.00 billion)")
@pytest.mark.parametrize('test_rounding', [False, True])
def test_readable_numbers_exhaustive(shell, test_rounding):
query = "select "
for i in range(1, 20):
if i > 1:
query += ", "
if test_rounding:
query += '9' * i
else:
query += '1' + ('0' * i)
test = (
ShellTest(shell)
.statement(".large_number_rendering all")
.statement(".maxwidth 99999")
.statement(query)
)
result = test.run()
for unit in ['million', 'billion', 'trillion', 'quadrillion', 'quintillion']:
for number in ['1.00', '10.00', '100.00']:
if unit == 'quintillion' and number in ['10.00', '100.00']:
continue
result.check_stdout(number + " " + unit)
def test_readable_numbers_rounding(shell):
test = (
ShellTest(shell)
.statement(".large_number_rendering footer")
.statement(".maxwidth 99999")
.statement("select 1005000, 1004999, -1005000, -1004999;")
)
result = test.run()
result.check_stdout("(1.01 million)")
result.check_stdout("(1.00 million)")
result.check_stdout("(-1.01 million)")
result.check_stdout("(-1.00 million)")
def test_readable_rounding_edge_case(shell):
test = (
ShellTest(shell)
.statement(".large_number_rendering all")
.statement(".maxwidth 99999")
.statement("select 994999, 995000")
)
result = test.run()
result.check_stdout("1.00 million")
def test_readable_numbers_limit(shell):
test = (
ShellTest(shell)
.statement(".maxwidth 99999")
.statement(".large_number_rendering all")
.statement("select 18446744073709551616, -18446744073709551616, 9999999999999999999, -9999999999999999999;")
)
result = test.run()
result.check_stdout("10.00 quintillion")
result.check_stdout("-10.00 quintillion")
def test_decimal_separator(shell):
test = (
ShellTest(shell)
.statement(".decimal_sep ,")
.statement(".large_number_rendering all")
.statement("select 59986052, 59986052.5, 999999999.123456789, 1e20, 'nan'::double;")
)
result = test.run()
result.check_stdout("59,99 million")
result.check_stdout("1,00 billion")
def test_odd_floating_points(shell):
test = (
ShellTest(shell)
.statement("select 1e20, 'nan'::double;")
)
result = test.run()
result.check_stdout("nan")
def test_disable_readable_numbers(shell):
test = (
ShellTest(shell)
.statement(".large_number_rendering off")
.statement("select 123456789;")
)
result = test.run()
result.check_not_exist('(123.46 million)')
def test_large_number_rendering_all(shell):
test = (
ShellTest(shell)
.statement(".large_number_rendering all")
.statement("select 123456789 from range(10);")
)
result = test.run()
result.check_stdout('123.46 million')
result.check_not_exist('(123.46 million)')
def test_readable_numbers_columns(shell):
test = (
ShellTest(shell)
.statement(".columns")
.statement("select 123456789;")
)
result = test.run()
result.check_not_exist('(123.46 million)')
def test_readable_numbers_row_count(shell):
test = (
ShellTest(shell)
.statement(".large_number_rendering footer")
.statement("select r from range(1230000) t(r);")
)
result = test.run()
result.check_stdout('1.23 million rows')
def test_readable_numbers_row_count_wide(shell):
test = (
ShellTest(shell)
.statement(".large_number_rendering footer")
.statement("select r, r, r, r, r, r, r from range(1230000) t(r);")
)
result = test.run()
result.check_stdout('1.23 million rows')
def test_readable_numbers_row_count_wide_single_col(shell):
test = (
ShellTest(shell)
.statement(".large_number_rendering footer")
.statement("select concat(r, r, r, r, r, r, r) c from range(1230000) t(r);")
)
result = test.run()
result.check_stdout('1.23 million rows')

65
external/duckdb/tools/shell/tests/test_safe_mode.py vendored Normal file
View File

@@ -0,0 +1,65 @@
# fmt: off
import pytest
import subprocess
import sys
from typing import List
from conftest import ShellTest
from tools.shell.tests.conftest import random_filepath
@pytest.mark.parametrize("command", [".sh ls", ".cd ..", ".log file", ".import file.csv tbl", ".open new_file", ".output out", ".once out", ".excel out", ".read myfile.sql"])
def test_safe_mode_command(shell, command):
test = (
ShellTest(shell, ['-safe'])
.statement(command)
)
result = test.run()
result.check_stderr('cannot be used in -safe mode')
@pytest.mark.parametrize("param", [(".sh ls", 'cannot be used in -safe mode'), ("INSTALL extension", "Permission Error")])
def test_safe_mode_dot_command(shell, param):
command = param[0]
expected_error = param[1]
test = (
ShellTest(shell)
.statement('.safe_mode')
.statement(command)
)
result = test.run()
result.check_stderr(expected_error)
def test_safe_mode_database_basic(shell, random_filepath):
test = (
ShellTest(shell, [random_filepath, '-safe'])
.statement('CREATE TABLE integers(i INT)')
.statement('INSERT INTO integers VALUES (1), (2), (3)')
.statement('SELECT SUM(i) FROM integers')
)
result = test.run()
result.check_stdout("6")
@pytest.mark.parametrize("command", [".sh ls", ".cd ..", ".log file", ".import file.csv tbl", ".open new_file", ".output out", ".once out", ".excel out", ".read myfile.sql"])
@pytest.mark.parametrize("persistent", [False, True])
def test_safe_mode_database_commands(shell, random_filepath, command, persistent):
arguments = ['-safe'] if not persistent else [random_filepath, '-safe']
test = (
ShellTest(shell, arguments)
.statement(command)
)
result = test.run()
result.check_stderr('cannot be used in -safe mode')
@pytest.mark.parametrize("sql", ["COPY (SELECT 42) TO 'test.csv'", "LOAD spatial", "INSTALL spatial", "ATTACH 'file.db' AS file"])
@pytest.mark.parametrize("persistent", [False, True])
def test_safe_mode_query(shell, random_filepath, sql, persistent):
arguments = ['-safe'] if not persistent else [random_filepath, '-safe']
test = (
ShellTest(shell, arguments)
.statement(sql)
)
result = test.run()
result.check_stderr('disabled')
# fmt: on

1211
external/duckdb/tools/shell/tests/test_shell_basics.py vendored Normal file
View File

File diff suppressed because it is too large Load Diff

51
external/duckdb/tools/shell/tests/test_shell_rendering.py vendored Normal file
View File

@@ -0,0 +1,51 @@
# fmt: off
import pytest
import subprocess
import sys
from typing import List
from conftest import ShellTest
import os
from pathlib import Path
def test_left_align(shell):
test = (
ShellTest(shell)
.statement(".mode box")
.statement(".width 5")
.statement(f'select 100 AS r')
)
result = test.run()
result.check_stdout("│ 100 │")
def test_right_align(shell):
test = (
ShellTest(shell)
.statement(".mode box")
.statement(".width -5")
.statement(f'select 100 AS r')
)
result = test.run()
result.check_stdout("│ 100 │")
def test_markdown(shell):
test = (
ShellTest(shell)
.statement(".mode markdown")
.statement("select 42 a, 'hello' str")
)
result = test.run()
result.check_stdout("| a | str |")
def test_mode_insert_table(shell):
test = (
ShellTest(shell)
.statement(".mode box mytable")
)
result = test.run()
result.check_stderr("TABLE argument can only be used with .mode insert")

47
external/duckdb/tools/shell/tests/test_temp_directory.py vendored Normal file
View File

@@ -0,0 +1,47 @@
# fmt: off
import pytest
import subprocess
import sys
from typing import List
from conftest import ShellTest
import os
def test_temp_directory(shell, tmp_path):
temp_dir = tmp_path / 'random_dir'
temp_dir.mkdir()
temp_file = temp_dir / 'myfile'
with open(temp_file, 'w+') as f:
f.write('hello world')
test = (
ShellTest(shell)
.statement(f"SET temp_directory='{temp_dir.as_posix()}';")
.statement("PRAGMA memory_limit='2MB';")
.statement("CREATE TABLE t1 AS SELECT * FROM range(1000000);")
)
result = test.run()
# make sure the temp directory or existing files are not deleted
assert os.path.isdir(temp_dir)
with open(temp_file, 'r') as f:
assert f.read() == "hello world"
# all other files are gone
assert os.listdir(temp_dir) == ['myfile']
os.remove(temp_file)
os.rmdir(temp_dir)
test = (
ShellTest(shell)
.statement(f"SET temp_directory='{temp_dir.as_posix()}';")
.statement("PRAGMA memory_limit='2MB';")
.statement("CREATE TABLE t1 AS SELECT * FROM range(1000000);")
)
result = test.run()
# make sure the temp directory is deleted
assert not os.path.exists(temp_dir)
assert not os.path.isdir(temp_dir)
# fmt: on