1003 lines
36 KiB
Python
1003 lines
36 KiB
Python
import os
|
|
import json
|
|
import re
|
|
import glob
|
|
import copy
|
|
import argparse
|
|
from packaging.version import Version
|
|
from functools import reduce
|
|
from pathlib import Path
|
|
|
|
###
|
|
# DuckDB C API header generation
|
|
###
|
|
# This script generates the DuckDB C API headers.
|
|
#
|
|
# The script works by parsing the definition files in `src/include/duckdb/main/capi/header_generation`, which are then
|
|
# used to generate the 3 header files:
|
|
#
|
|
# The main C header. This is what to include when linking with DuckDB through the C API.
|
|
DUCKDB_HEADER_OUT_FILE_NAME = 'duckdb.h'
|
|
DUCKDB_HEADER_OUT_FILE = 'src/include/' + DUCKDB_HEADER_OUT_FILE_NAME
|
|
# The header to be included by DuckDB C extensions.
|
|
DUCKDB_HEADER_C_OUT_FILE = 'src/include/duckdb_extension.h'
|
|
# The header to be included by DuckDB Go extensions.
|
|
DUCKDB_HEADER_GO_OUT_FILE = 'src/include/duckdb_go_extension.h'
|
|
# An internal header for DuckDB C extensions.
|
|
DUCKDB_HEADER_EXT_INTERNAL_OUT_FILE = 'src/include/duckdb/main/capi/extension_api.hpp'
|
|
|
|
# Whether the script allows functions with parameters without a comment explaining them
|
|
ALLOW_UNCOMMENTED_PARAMS = True
|
|
|
|
DUCKDB_EXT_API_VAR_NAME = 'duckdb_ext_api'
|
|
DUCKDB_EXT_API_STRUCT_TYPENAME = 'duckdb_ext_api_v1'
|
|
|
|
DEV_VERSION_TAG = 'unstable'
|
|
|
|
# Define the extension struct
|
|
EXT_API_DEFINITION_PATTERN = 'src/include/duckdb/main/capi/header_generation/apis/v1/*/*.json'
|
|
EXT_API_EXCLUSION_FILE = 'src/include/duckdb/main/capi/header_generation/apis/v1/exclusion_list.json'
|
|
|
|
# The JSON files that define all available CAPI functions
|
|
CAPI_FUNCTION_DEFINITION_FILES = 'src/include/duckdb/main/capi/header_generation/functions/**/*.json'
|
|
|
|
# The original order of the function groups in the duckdb.h files. We maintain this for easier PR reviews.
|
|
# TODO: replace this with alphabetical ordering in a separate PR
|
|
ORIGINAL_FUNCTION_GROUP_ORDER = [
|
|
'open_connect',
|
|
'configuration',
|
|
'error_data',
|
|
'query_execution',
|
|
'result_functions',
|
|
'safe_fetch_functions',
|
|
'helpers',
|
|
'date_time_timestamp_helpers',
|
|
'hugeint_helpers',
|
|
'unsigned_hugeint_helpers',
|
|
'decimal_helpers',
|
|
'prepared_statements',
|
|
'bind_values_to_prepared_statements',
|
|
'execute_prepared_statements',
|
|
'extract_statements',
|
|
'pending_result_interface',
|
|
'value_interface',
|
|
'logical_type_interface',
|
|
'data_chunk_interface',
|
|
'vector_interface',
|
|
'validity_mask_functions',
|
|
'scalar_functions',
|
|
'selection_vector_interface',
|
|
'aggregate_functions',
|
|
'table_functions',
|
|
'table_function_bind',
|
|
'table_function_init',
|
|
'table_function',
|
|
'replacement_scans',
|
|
'profiling_info',
|
|
'appender',
|
|
'table_description',
|
|
'arrow_interface',
|
|
'threading_information',
|
|
'streaming_result_interface',
|
|
'cast_functions',
|
|
'expression_interface',
|
|
'file_system_interface',
|
|
]
|
|
|
|
# The file that forms the base for the header generation
|
|
BASE_HEADER_TEMPLATE = 'src/include/duckdb/main/capi/header_generation/header_base.hpp.template'
|
|
# The comment marking where this script will inject its contents
|
|
BASE_HEADER_CONTENT_MARK = '// FILE_CONTENT_SECTION\n'
|
|
|
|
|
|
def HEADER(file):
|
|
return f'''//===----------------------------------------------------------------------===//
|
|
//
|
|
// DuckDB
|
|
//
|
|
// {file}
|
|
//
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// !!!!!!!
|
|
// WARNING: this file is autogenerated by scripts/generate_c_api.py, manual changes will be overwritten
|
|
// !!!!!!!
|
|
'''
|
|
|
|
|
|
def COMMENT_HEADER(name):
|
|
return f'''//===--------------------------------------------------------------------===//
|
|
// {name}
|
|
//===--------------------------------------------------------------------===//
|
|
'''
|
|
|
|
|
|
HELPER_MACROS = f'''
|
|
#ifdef __cplusplus
|
|
#define DUCKDB_EXTENSION_EXTERN_C_GUARD_OPEN extern "C" {{
|
|
#define DUCKDB_EXTENSION_EXTERN_C_GUARD_CLOSE }}
|
|
#else
|
|
#define DUCKDB_EXTENSION_EXTERN_C_GUARD_OPEN
|
|
#define DUCKDB_EXTENSION_EXTERN_C_GUARD_CLOSE
|
|
#endif
|
|
|
|
#define DUCKDB_EXTENSION_GLUE_HELPER(x, y) x##y
|
|
#define DUCKDB_EXTENSION_GLUE(x, y) DUCKDB_EXTENSION_GLUE_HELPER(x, y)
|
|
#define DUCKDB_EXTENSION_STR_HELPER(x) #x
|
|
#define DUCKDB_EXTENSION_STR(x) DUCKDB_EXTENSION_STR_HELPER(x)
|
|
#define DUCKDB_EXTENSION_SEMVER_STRING(major, minor, patch) "v" DUCKDB_EXTENSION_STR_HELPER(major) "." DUCKDB_EXTENSION_STR_HELPER(minor) "." DUCKDB_EXTENSION_STR_HELPER(patch)
|
|
|
|
'''
|
|
|
|
DUCKDB_H_HEADER = HEADER(DUCKDB_HEADER_OUT_FILE_NAME)
|
|
DUCKDB_C_H_HEADER = HEADER('duckdb_extension.h')
|
|
DUCKDB_GO_H_HEADER = HEADER('duckdb_go_extension.h')
|
|
DUCKDB_EXT_INTERNAL_H_HEADER = HEADER('extension_api.hpp')
|
|
|
|
UNSTABLE_WARNING = "\n// WARNING: this API is not yet stable, meaning it is only guaranteed to work for this specific DuckDB version.\n\n"
|
|
DUCKDB_C_H_HEADER += UNSTABLE_WARNING
|
|
DUCKDB_GO_H_HEADER += UNSTABLE_WARNING
|
|
|
|
|
|
# Loads the header file template.
|
|
def fetch_header_template_main():
|
|
# Read the base header
|
|
with open(BASE_HEADER_TEMPLATE, 'r') as f:
|
|
result = f.read()
|
|
|
|
# Trim the base header
|
|
header_mark = '// DUCKDB_START_OF_HEADER\n'
|
|
if header_mark not in result:
|
|
print(f"Could not find the header start mark: {header_mark}")
|
|
exit(1)
|
|
|
|
return result[result.find(header_mark) + len(header_mark) :]
|
|
|
|
|
|
# Includes the header file template.
|
|
def fetch_header_template_ext(header):
|
|
return f"""#pragma once
|
|
|
|
#include "{header}"
|
|
|
|
"""
|
|
|
|
|
|
# Parse the CAPI_FUNCTION_DEFINITION_FILES to get the full list of functions
|
|
def parse_capi_function_definitions():
|
|
# Collect all functions
|
|
function_files = glob.glob(CAPI_FUNCTION_DEFINITION_FILES, recursive=True)
|
|
|
|
function_groups = []
|
|
function_map = {}
|
|
|
|
# Read functions
|
|
for file in function_files:
|
|
with open(file, 'r') as f:
|
|
try:
|
|
json_data = json.loads(f.read())
|
|
except json.decoder.JSONDecodeError as err:
|
|
print(f"Invalid JSON found in {file}: {err}")
|
|
exit(1)
|
|
|
|
function_groups.append(json_data)
|
|
for function in json_data['entries']:
|
|
if function['name'] in function_map:
|
|
print(f"Duplicate symbol found when parsing C API file {file}: {function['name']}")
|
|
exit(1)
|
|
|
|
function['group'] = json_data['group']
|
|
if 'deprecated' in json_data:
|
|
function['group_deprecated'] = json_data['deprecated']
|
|
|
|
function_map[function['name']] = function
|
|
|
|
# Reorder to match original order: purely intended to keep the PR review sane
|
|
function_groups_ordered = []
|
|
|
|
if len(function_groups) != len(ORIGINAL_FUNCTION_GROUP_ORDER):
|
|
print(
|
|
"The list used to match the original order of function groups in the original the duckdb.h file does not match the new one. Did you add a new function group? please also add it to ORIGINAL_FUNCTION_GROUP_ORDER for now."
|
|
)
|
|
|
|
for order_group in ORIGINAL_FUNCTION_GROUP_ORDER:
|
|
curr_group = next(group for group in function_groups if group['group'] == order_group)
|
|
function_groups.remove(curr_group)
|
|
function_groups_ordered.append(curr_group)
|
|
|
|
return function_groups_ordered, function_map
|
|
|
|
|
|
# Read extension API definitions.
|
|
def parse_ext_api_definitions(ext_api_definition):
|
|
api_definitions = {}
|
|
versions = []
|
|
dev_versions = []
|
|
for file in list(glob.glob(ext_api_definition)):
|
|
with open(file, 'r') as f:
|
|
try:
|
|
obj = json.loads(f.read())
|
|
api_definitions[obj['version']] = obj
|
|
if obj['version'].startswith("unstable_"):
|
|
dev_versions.append(obj['version'])
|
|
else:
|
|
if Path(file).stem != obj['version']:
|
|
print(
|
|
f"\nMismatch between filename and version in file for {file}. Note that unstable versions should have a version starting with 'unstable_' and that stable versions should have the version as their filename"
|
|
)
|
|
exit(1)
|
|
versions.append(obj['version'])
|
|
|
|
except json.decoder.JSONDecodeError as err:
|
|
print(f"\nInvalid JSON found in {file}: {err}")
|
|
exit(1)
|
|
|
|
versions.sort(key=Version)
|
|
dev_versions.sort()
|
|
|
|
return [api_definitions[x] for x in (versions + dev_versions)]
|
|
|
|
|
|
# Decode all excluded functions and return them.
|
|
def parse_exclusion_list(function_map):
|
|
exclusion_set = set()
|
|
with open(EXT_API_EXCLUSION_FILE, 'r') as f:
|
|
try:
|
|
data = json.loads(f.read())
|
|
except json.decoder.JSONDecodeError as err:
|
|
print(f"\nInvalid JSON found in {EXT_API_EXCLUSION_FILE}: {err}")
|
|
exit(1)
|
|
|
|
for group in data['exclusion_list']:
|
|
for entry in group['entries']:
|
|
if entry not in function_map:
|
|
print(f"\nInvalid item found in exclusion list: {entry}. This entry does not occur in the API!")
|
|
exit(1)
|
|
exclusion_set.add(entry)
|
|
return exclusion_set
|
|
|
|
|
|
# Creates the comment that accompanies a C API function.
|
|
def create_function_comment(function_obj):
|
|
result = ''
|
|
function_name = function_obj['name']
|
|
|
|
# Construct comment
|
|
if 'comment' in function_obj:
|
|
comment = function_obj['comment']
|
|
result += '/*!\n'
|
|
result += comment['description']
|
|
|
|
# result += '\n\n'
|
|
if 'params' in function_obj:
|
|
for param in function_obj['params']:
|
|
param_name = param['name']
|
|
if not 'param_comments' in comment:
|
|
if not ALLOW_UNCOMMENTED_PARAMS:
|
|
print(comment)
|
|
print(f'\nMissing param comments for function {function_name}')
|
|
exit(1)
|
|
continue
|
|
if param['name'] in comment['param_comments']:
|
|
param_comment = comment['param_comments'][param['name']]
|
|
result += f'* @param {param_name} {param_comment}\n'
|
|
elif not ALLOW_UNCOMMENTED_PARAMS:
|
|
print(f'\nUncommented parameter found: {param_name} of function {function_name}')
|
|
exit(1)
|
|
|
|
if 'return_value' in comment:
|
|
comment_return_value = comment['return_value']
|
|
result += f'* @return {comment_return_value}\n'
|
|
result += '*/\n'
|
|
return result
|
|
|
|
|
|
# Creates the function declaration for the regular C header file
|
|
def create_function_declaration(function_obj):
|
|
result = ''
|
|
function_name = function_obj['name']
|
|
function_return_type = function_obj['return_type']
|
|
|
|
# Construct function declaration
|
|
result += f'DUCKDB_C_API {function_return_type}'
|
|
if result[-1] != '*':
|
|
result += ' '
|
|
result += f'{function_name}('
|
|
|
|
if 'params' in function_obj:
|
|
if len(function_obj['params']) > 0:
|
|
for param in function_obj['params']:
|
|
param_type = param['type']
|
|
param_name = param['name']
|
|
result += f'{param_type}'
|
|
if result[-1] != '*':
|
|
result += ' '
|
|
result += f'{param_name}, '
|
|
result = result[:-2] # Trailing comma
|
|
result += ');\n'
|
|
|
|
return result
|
|
|
|
|
|
# Creates the function declaration for extension api struct
|
|
def create_struct_member(function_obj):
|
|
result = ''
|
|
|
|
function_name = function_obj['name']
|
|
function_return_type = function_obj['return_type']
|
|
result += f' {function_return_type} (*{function_name})('
|
|
if 'params' in function_obj:
|
|
if len(function_obj['params']) > 0:
|
|
for param in function_obj['params']:
|
|
param_type = param['type']
|
|
param_name = param['name']
|
|
result += f'{param_type} {param_name},'
|
|
result = result[:-1] # Trailing comma
|
|
result += ');'
|
|
|
|
return result
|
|
|
|
|
|
# Creates the function declaration for extension api struct
|
|
def create_function_typedef(function_obj):
|
|
function_name = function_obj['name']
|
|
return f'#define {function_name} {DUCKDB_EXT_API_VAR_NAME}.{function_name}\n'
|
|
|
|
|
|
def headline_capitalize(s, i):
|
|
if i > 0 and s in [
|
|
"a",
|
|
"an",
|
|
"the",
|
|
"and",
|
|
"but",
|
|
"or",
|
|
"nor",
|
|
"for",
|
|
"so",
|
|
"yet",
|
|
"about",
|
|
"above",
|
|
"across",
|
|
"after",
|
|
"against",
|
|
"along",
|
|
"among",
|
|
"around",
|
|
"at",
|
|
"before",
|
|
"behind",
|
|
"below",
|
|
"beneath",
|
|
"beside",
|
|
"between",
|
|
"beyond",
|
|
"by",
|
|
"despite",
|
|
"down",
|
|
"during",
|
|
"except",
|
|
"for",
|
|
"from",
|
|
"in",
|
|
"inside",
|
|
"into",
|
|
"like",
|
|
"near",
|
|
"of",
|
|
"off",
|
|
"on",
|
|
"onto",
|
|
"out",
|
|
"outside",
|
|
"over",
|
|
"past",
|
|
"since",
|
|
"through",
|
|
"throughout",
|
|
"to",
|
|
"toward",
|
|
"under",
|
|
"underneath",
|
|
"until",
|
|
"up",
|
|
"upon",
|
|
"with",
|
|
"within",
|
|
"without",
|
|
]:
|
|
return s
|
|
else:
|
|
return s.capitalize()
|
|
|
|
|
|
def to_camel_case(snake_str):
|
|
return " ".join(headline_capitalize(s, i) for i, s in enumerate(snake_str.lower().split("_")))
|
|
|
|
|
|
def parse_semver(version):
|
|
if version[0] != 'v':
|
|
raise Exception(f"\nVersion string {version} does not start with a v")
|
|
|
|
versions = version[1:].split(".")
|
|
|
|
if len(versions) != 3:
|
|
print(f"\nVersion string {version} is invalid, only vx.y.z is supported")
|
|
exit(1)
|
|
|
|
return int(versions[0]), int(versions[1]), int(versions[2])
|
|
|
|
|
|
def create_version_defines(version):
|
|
major, minor, patch = parse_semver(version)
|
|
|
|
version_string = f"v{major}.{minor}.{patch}"
|
|
|
|
result = ""
|
|
result += f"#define DUCKDB_EXTENSION_API_VERSION_MAJOR {major}\n"
|
|
result += f"#define DUCKDB_EXTENSION_API_VERSION_MINOR {minor}\n"
|
|
result += f"#define DUCKDB_EXTENSION_API_VERSION_PATCH {patch}\n"
|
|
result += f"#define DUCKDB_EXTENSION_API_VERSION_STRING \"{version_string}\"\n"
|
|
|
|
return result
|
|
|
|
|
|
# Create duckdb.h
|
|
def create_duckdb_h(file, function_groups, write_functions=True):
|
|
declarations = ''
|
|
|
|
if write_functions:
|
|
declarations += f'#ifndef DUCKDB_API_EXCLUDE_FUNCTIONS\n\n'
|
|
declarations += COMMENT_HEADER("Functions")
|
|
declarations += '\n'
|
|
|
|
for curr_group in function_groups:
|
|
declarations += COMMENT_HEADER(to_camel_case(curr_group['group']))
|
|
declarations += '\n'
|
|
if 'description' in curr_group:
|
|
declarations += curr_group['description'] + '\n'
|
|
|
|
deprecated_state = False
|
|
group_is_deprecated = 'deprecated' in curr_group and curr_group['deprecated']
|
|
if group_is_deprecated:
|
|
declarations += f'#ifndef DUCKDB_API_NO_DEPRECATED\n'
|
|
deprecated_state = True
|
|
|
|
for function in curr_group['entries']:
|
|
function_is_deprecated = group_is_deprecated or ('deprecated' in function and function['deprecated'])
|
|
if deprecated_state and not function_is_deprecated:
|
|
declarations += '#endif\n\n'
|
|
deprecated_state = False
|
|
elif not deprecated_state and function_is_deprecated:
|
|
declarations += '#ifndef DUCKDB_API_NO_DEPRECATED\n'
|
|
deprecated_state = True
|
|
|
|
function_comment = create_function_comment(function)
|
|
function_deprecation_notice = (
|
|
'**DEPRECATED**' in function_comment or '**DEPRECATION NOTICE**' in function_comment
|
|
)
|
|
if function_is_deprecated and not function_deprecation_notice:
|
|
raise Exception(
|
|
f"Function {str(function)} is labeled as deprecated but the comment does not indicate this"
|
|
)
|
|
elif not function_is_deprecated and function_deprecation_notice:
|
|
raise Exception(
|
|
f"Function {str(function)} is not labeled as deprecated but the comment indicates that it is"
|
|
)
|
|
declarations += function_comment
|
|
declarations += create_function_declaration(function)
|
|
declarations += '\n'
|
|
|
|
if deprecated_state:
|
|
declarations += '#endif\n\n'
|
|
|
|
declarations += '#endif\n'
|
|
|
|
# Write the function declarations to the header template, and then write the file.
|
|
header_template = fetch_header_template_main()
|
|
duckdb_h = DUCKDB_H_HEADER + header_template.replace(BASE_HEADER_CONTENT_MARK, declarations)
|
|
with open(file, 'w+') as f:
|
|
f.write(duckdb_h)
|
|
|
|
|
|
def write_struct_member_definitions(function_map, version_entries, initialize=False):
|
|
result = ""
|
|
if initialize:
|
|
for function_name in version_entries:
|
|
function_lookup = function_map[function_name]
|
|
function_lookup_name = function_lookup['name']
|
|
result += f' result.{function_lookup_name} = {function_lookup_name};\n'
|
|
elif len(version_entries) > 0:
|
|
count = len(version_entries)
|
|
first_function = version_entries[0]
|
|
result += f' memset(&result.{first_function}, 0, sizeof(result.{first_function}) * {count});\n'
|
|
|
|
return result
|
|
|
|
|
|
# Creates function pointer invokers.
|
|
def create_struct_member_invoker(function_obj):
|
|
result = ''
|
|
function_name = function_obj['name']
|
|
function_return_type = function_obj['return_type']
|
|
|
|
# Construct typedef declaration
|
|
result += f'static {function_return_type}'
|
|
if result[-1] != '*':
|
|
result += ' '
|
|
result += f'{function_name}('
|
|
invoked_params = ''
|
|
|
|
if 'params' in function_obj:
|
|
if len(function_obj['params']) > 0:
|
|
for param in function_obj['params']:
|
|
param_type = param['type']
|
|
param_name = param['name']
|
|
result += f'{param_type}'
|
|
if result[-1] != '*':
|
|
result += ' '
|
|
result += f'{param_name}, '
|
|
invoked_params += f'{param_name}, '
|
|
result = result[:-2] # Trailing comma
|
|
result += ') {\n'
|
|
|
|
if invoked_params != '':
|
|
invoked_params = invoked_params[:-2] # Trailing comma
|
|
result += f'\treturn duckdb_ext_api->{function_name}({invoked_params}); \n'
|
|
result += '}'
|
|
return result
|
|
|
|
|
|
def create_struct_version_defines(api_definition):
|
|
result = "//! These defines control which version of the API is available\n"
|
|
|
|
for i in range(1, len(api_definition) + 1):
|
|
current_version = api_definition[-i]['version']
|
|
if i < len(api_definition):
|
|
prev_version = api_definition[-(i + 1)]['version']
|
|
else:
|
|
prev_version = None
|
|
|
|
print(f"current: {current_version} prev: {prev_version}")
|
|
|
|
if prev_version:
|
|
prev_major, prev_minor, prev_patch = parse_semver(prev_version)
|
|
if current_version == DEV_VERSION_TAG:
|
|
result += "#ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE\n"
|
|
result += f"#define DUCKDB_EXTENSION_API_VERSION_{prev_major}_{prev_minor}_{prev_patch}\n"
|
|
result += "#endif\n"
|
|
else:
|
|
major, minor, patch = parse_semver(current_version)
|
|
result += f"#ifdef DUCKDB_EXTENSION_API_VERSION_{major}_{minor}_{patch}\n"
|
|
result += f"#define DUCKDB_EXTENSION_API_VERSION_{prev_major}_{prev_minor}_{prev_patch}\n"
|
|
result += "#endif\n"
|
|
result += "\n"
|
|
|
|
first_version = api_definition[0]['version']
|
|
first_major, first_minor, first_patch = parse_semver(first_version)
|
|
result += f"// No version was explicitly set, we assume latest\n"
|
|
result += f"#ifndef DUCKDB_EXTENSION_API_VERSION_{first_major}_{first_minor}_{first_patch}\n"
|
|
result += f"#define DUCKDB_EXTENSION_API_VERSION_LATEST\n"
|
|
result += f"#endif\n"
|
|
|
|
result += "\n"
|
|
return result
|
|
|
|
|
|
# Generate the struct without any other content.
|
|
def generate_basic_extension_struct(function_map, api_definition, add_version_defines):
|
|
functions_in_struct = set()
|
|
extension_struct_finished = COMMENT_HEADER("Function pointer struct")
|
|
extension_struct_finished += 'typedef struct {\n'
|
|
|
|
for api_version_entry in api_definition:
|
|
if len(api_version_entry['entries']) == 0:
|
|
continue
|
|
version = api_version_entry['version']
|
|
if version.startswith("unstable_"):
|
|
if 'description' in api_version_entry:
|
|
extension_struct_finished += f"// {api_version_entry['description']}\n"
|
|
if add_version_defines:
|
|
extension_struct_finished += f"#ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE\n"
|
|
else:
|
|
extension_struct_finished += f"\n"
|
|
else:
|
|
if add_version_defines:
|
|
major, minor, patch = parse_semver(version)
|
|
extension_struct_finished += f"#if DUCKDB_EXTENSION_API_VERSION_MINOR > {minor} || (DUCKDB_EXTENSION_API_VERSION_MINOR == {minor} && DUCKDB_EXTENSION_API_VERSION_PATCH >= {patch}) // {version}\n"
|
|
else:
|
|
extension_struct_finished += f"// {version}\n"
|
|
|
|
for function_name in api_version_entry['entries']:
|
|
function_lookup = function_map[function_name]
|
|
functions_in_struct.add(function_lookup['name'])
|
|
extension_struct_finished += create_struct_member(function_lookup)
|
|
extension_struct_finished += '\n'
|
|
if add_version_defines:
|
|
extension_struct_finished += "#endif\n\n"
|
|
|
|
extension_struct_finished += '} ' + f'{DUCKDB_EXT_API_STRUCT_TYPENAME};\n\n'
|
|
return extension_struct_finished, functions_in_struct
|
|
|
|
|
|
# Check for missing and duplicate entries.
|
|
def struct_validate_exclusion_list(functions_in_struct, api_definition, exclusion_set, function_groups):
|
|
missing_entries = []
|
|
for group in function_groups:
|
|
for function in group['entries']:
|
|
if function['name'] not in functions_in_struct and function['name'] not in exclusion_set:
|
|
missing_entries.append(function['name'])
|
|
if missing_entries:
|
|
print(
|
|
"\nExclusion list validation failed! This means a C API function has been defined but not added to the API struct nor the exclusion list"
|
|
)
|
|
print(f" * Missing functions: {missing_entries}")
|
|
exit(1)
|
|
|
|
double_entries = []
|
|
for api_version_entry in api_definition:
|
|
for function_name in api_version_entry['entries']:
|
|
if function_name in exclusion_set:
|
|
double_entries.append(function_name)
|
|
if double_entries:
|
|
print(
|
|
"\nExclusion list is invalid, there are entries in the extension api that are also in the exclusion list!"
|
|
)
|
|
print(f" * Missing functions: {double_entries}")
|
|
exit(1)
|
|
|
|
|
|
def create_extension_api_struct(
|
|
function_groups,
|
|
function_map,
|
|
api_definition,
|
|
exclusion_set,
|
|
with_create_method=False,
|
|
with_member_invoker=False,
|
|
add_version_defines=False,
|
|
create_method_name='',
|
|
validate_exclusion_list=True,
|
|
):
|
|
extension_struct_finished, functions_in_struct = generate_basic_extension_struct(
|
|
function_map, api_definition, add_version_defines
|
|
)
|
|
|
|
if validate_exclusion_list:
|
|
struct_validate_exclusion_list(functions_in_struct, api_definition, exclusion_set, function_groups)
|
|
|
|
if with_create_method:
|
|
extension_struct_finished += COMMENT_HEADER("Struct Create Method")
|
|
extension_struct_finished += f"inline {DUCKDB_EXT_API_STRUCT_TYPENAME} {create_method_name}() {{\n"
|
|
extension_struct_finished += f" {DUCKDB_EXT_API_STRUCT_TYPENAME} result;\n"
|
|
|
|
for api_version_entry in api_definition:
|
|
if len(api_version_entry['entries']) == 0:
|
|
continue
|
|
extension_struct_finished += write_struct_member_definitions(
|
|
function_map, api_version_entry['entries'], initialize=True
|
|
)
|
|
|
|
extension_struct_finished += " return result;\n"
|
|
extension_struct_finished += "}\n\n"
|
|
|
|
if with_member_invoker:
|
|
extension_struct_finished += f'extern {DUCKDB_EXT_API_STRUCT_TYPENAME} *{DUCKDB_EXT_API_VAR_NAME};\n\n'
|
|
extension_struct_finished += COMMENT_HEADER("Invoker Functions")
|
|
|
|
for api_version_entry in api_definition:
|
|
if len(api_version_entry['entries']) == 0:
|
|
continue
|
|
|
|
extension_struct_finished += '\n'
|
|
version = api_version_entry['version']
|
|
if version.startswith("unstable_"):
|
|
if add_version_defines:
|
|
extension_struct_finished += f"#ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE\n"
|
|
else:
|
|
extension_struct_finished += f"\n"
|
|
else:
|
|
if add_version_defines:
|
|
major, minor, patch = parse_semver(version)
|
|
extension_struct_finished += f"#if DUCKDB_EXTENSION_API_VERSION_MINOR > {minor} || (DUCKDB_EXTENSION_API_VERSION_MINOR == {minor} && DUCKDB_EXTENSION_API_VERSION_PATCH >= {patch}) // {version}\n"
|
|
else:
|
|
extension_struct_finished += f"// {version}\n"
|
|
extension_struct_finished += '\n'
|
|
|
|
for function_name in api_version_entry['entries']:
|
|
function_lookup = function_map[function_name]
|
|
functions_in_struct.add(function_lookup['name'])
|
|
extension_struct_finished += create_struct_member_invoker(function_lookup)
|
|
extension_struct_finished += '\n\n'
|
|
if add_version_defines:
|
|
extension_struct_finished += "#endif\n\n"
|
|
|
|
return extension_struct_finished
|
|
|
|
|
|
def create_duckdb_ext_h_versioning(
|
|
ext_api_version,
|
|
function_groups,
|
|
api_struct_definition,
|
|
exclusion_set,
|
|
duckdb_header,
|
|
duckdb_ext_header,
|
|
with_member_invoker=False,
|
|
):
|
|
|
|
# Create the versioning defines
|
|
major, minor, patch = parse_semver(ext_api_version)
|
|
versioning_defines = f"""//! Set version to latest if no explicit version is defined
|
|
|
|
#if !defined(DUCKDB_EXTENSION_API_VERSION_MAJOR) && !defined(DUCKDB_EXTENSION_API_VERSION_MINOR) && !defined(DUCKDB_EXTENSION_API_VERSION_PATCH)
|
|
#define DUCKDB_EXTENSION_API_VERSION_MAJOR {major}
|
|
#define DUCKDB_EXTENSION_API_VERSION_MINOR {minor}
|
|
#define DUCKDB_EXTENSION_API_VERSION_PATCH {patch}
|
|
#elif !(defined(DUCKDB_EXTENSION_API_VERSION_MAJOR) && defined(DUCKDB_EXTENSION_API_VERSION_MINOR) && defined(DUCKDB_EXTENSION_API_VERSION_PATCH))
|
|
#error "either all or none of the DUCKDB_EXTENSION_API_VERSION_ defines should be defined"
|
|
#endif
|
|
|
|
//! Set the DUCKDB_EXTENSION_API_VERSION_STRING which is passed to DuckDB on extension load
|
|
#ifdef DUCKDB_EXTENSION_API_UNSTABLE_VERSION
|
|
#define DUCKDB_EXTENSION_API_VERSION_STRING DUCKDB_EXTENSION_API_UNSTABLE_VERSION
|
|
#else
|
|
#define DUCKDB_EXTENSION_API_VERSION_STRING DUCKDB_EXTENSION_SEMVER_STRING(DUCKDB_EXTENSION_API_VERSION_MAJOR, DUCKDB_EXTENSION_API_VERSION_MINOR, DUCKDB_EXTENSION_API_VERSION_PATCH)
|
|
#endif
|
|
|
|
#if DUCKDB_EXTENSION_API_VERSION_MAJOR != {major}
|
|
#error "This version of the extension API header only supports API VERSION v{major}.x.x"
|
|
#endif
|
|
"""
|
|
|
|
# Begin constructing the header file
|
|
duckdb_ext_h = ""
|
|
duckdb_ext_h += duckdb_ext_header
|
|
duckdb_ext_h += fetch_header_template_ext(duckdb_header)
|
|
duckdb_ext_h += COMMENT_HEADER("Util Macros")
|
|
duckdb_ext_h += HELPER_MACROS
|
|
|
|
duckdb_ext_h += COMMENT_HEADER("Versioning")
|
|
duckdb_ext_h += versioning_defines
|
|
duckdb_ext_h += "\n"
|
|
|
|
duckdb_ext_h += create_extension_api_struct(
|
|
function_groups,
|
|
function_map,
|
|
api_struct_definition,
|
|
exclusion_set,
|
|
add_version_defines=True,
|
|
with_member_invoker=with_member_invoker,
|
|
)
|
|
duckdb_ext_h += "\n\n"
|
|
return duckdb_ext_h
|
|
|
|
|
|
# Create duckdb_extension.h
|
|
def create_duckdb_c_ext_h(file, ext_api_version, function_groups, api_struct_definition, exclusion_set):
|
|
duckdb_ext_h = create_duckdb_ext_h_versioning(
|
|
ext_api_version,
|
|
function_groups,
|
|
api_struct_definition,
|
|
exclusion_set,
|
|
DUCKDB_HEADER_OUT_FILE_NAME,
|
|
DUCKDB_C_H_HEADER,
|
|
)
|
|
|
|
# Generate the typedefs
|
|
typedefs = ""
|
|
for api_version_entry in api_struct_definition:
|
|
version = api_version_entry['version']
|
|
|
|
# Collect the typedefs for this version
|
|
grouped_typedefs = []
|
|
for group in function_groups:
|
|
functions_to_add = []
|
|
for function in group['entries']:
|
|
if function['name'] not in api_version_entry['entries']:
|
|
continue
|
|
functions_to_add.append(function)
|
|
|
|
if functions_to_add:
|
|
grouped_typedefs.append((group['group'], functions_to_add))
|
|
|
|
if len(grouped_typedefs) == 0:
|
|
continue
|
|
|
|
typedefs += f"// Version {version}\n"
|
|
|
|
for group in grouped_typedefs:
|
|
for fun_to_add in group[1]:
|
|
typedefs += create_function_typedef(fun_to_add)
|
|
|
|
typedefs += '\n'
|
|
|
|
duckdb_ext_h += COMMENT_HEADER("Typedefs mapping functions to struct entries")
|
|
duckdb_ext_h += typedefs
|
|
duckdb_ext_h += "\n"
|
|
|
|
# Add The struct global macros
|
|
duckdb_ext_h += COMMENT_HEADER("Struct Global Macros")
|
|
duckdb_ext_h += f"""// This goes in the c/c++ file containing the entrypoint (handle
|
|
#define DUCKDB_EXTENSION_GLOBAL {DUCKDB_EXT_API_STRUCT_TYPENAME} {DUCKDB_EXT_API_VAR_NAME} = {{0}};
|
|
// Initializes the C Extension API: First thing to call in the extension entrypoint
|
|
#define DUCKDB_EXTENSION_API_INIT(info, access, minimum_api_version) {DUCKDB_EXT_API_STRUCT_TYPENAME} * res = ({DUCKDB_EXT_API_STRUCT_TYPENAME} *)access->get_api(info, minimum_api_version); if (!res) {{return false;}}; {DUCKDB_EXT_API_VAR_NAME} = *res;
|
|
"""
|
|
duckdb_ext_h += f"""
|
|
// Place in global scope of any C/C++ file that needs to access the extension API
|
|
#define DUCKDB_EXTENSION_EXTERN extern {DUCKDB_EXT_API_STRUCT_TYPENAME} {DUCKDB_EXT_API_VAR_NAME};
|
|
|
|
#ifdef _WIN32
|
|
#define DUCKDB_CAPI_ENTRY_VISIBILITY __declspec(dllexport)
|
|
#else
|
|
#define DUCKDB_CAPI_ENTRY_VISIBILITY __attribute__((visibility("default")))
|
|
#endif
|
|
|
|
"""
|
|
# Add the entrypoint macros
|
|
duckdb_ext_h += COMMENT_HEADER("Entrypoint Macros")
|
|
duckdb_ext_h += """
|
|
// Note: the DUCKDB_EXTENSION_ENTRYPOINT macro requires DUCKDB_EXTENSION_NAME to be set.
|
|
|
|
#ifdef DUCKDB_EXTENSION_NAME
|
|
|
|
// Main entrypoint: opens (and closes) a connection automatically for the extension to register its functionality through
|
|
#define DUCKDB_EXTENSION_ENTRYPOINT\
|
|
DUCKDB_EXTENSION_GLOBAL static bool DUCKDB_EXTENSION_GLUE(DUCKDB_EXTENSION_NAME,_init_c_api_internal)(duckdb_connection connection, duckdb_extension_info info, struct duckdb_extension_access *access);\
|
|
DUCKDB_EXTENSION_EXTERN_C_GUARD_OPEN\
|
|
DUCKDB_CAPI_ENTRY_VISIBILITY DUCKDB_EXTENSION_API bool DUCKDB_EXTENSION_GLUE(DUCKDB_EXTENSION_NAME,_init_c_api)(\
|
|
duckdb_extension_info info, struct duckdb_extension_access *access) {\
|
|
DUCKDB_EXTENSION_API_INIT(info, access, DUCKDB_EXTENSION_API_VERSION_STRING);\
|
|
duckdb_database *db = access->get_database(info);\
|
|
duckdb_connection conn;\
|
|
if (duckdb_connect(*db, &conn) == DuckDBError) {\
|
|
access->set_error(info, "Failed to open connection to database");\
|
|
return false;\
|
|
}\
|
|
bool init_result = DUCKDB_EXTENSION_GLUE(DUCKDB_EXTENSION_NAME,_init_c_api_internal)(conn, info, access);\
|
|
duckdb_disconnect(&conn);\
|
|
return init_result;\
|
|
}\
|
|
DUCKDB_EXTENSION_EXTERN_C_GUARD_CLOSE static bool DUCKDB_EXTENSION_GLUE(DUCKDB_EXTENSION_NAME,_init_c_api_internal)
|
|
|
|
// Custom entrypoint: just forwards the info and access
|
|
#define DUCKDB_EXTENSION_ENTRYPOINT_CUSTOM\
|
|
DUCKDB_EXTENSION_GLOBAL static bool DUCKDB_EXTENSION_GLUE(DUCKDB_EXTENSION_NAME,_init_c_api_internal)(\
|
|
duckdb_extension_info info, struct duckdb_extension_access *access);\
|
|
DUCKDB_EXTENSION_EXTERN_C_GUARD_OPEN\
|
|
DUCKDB_CAPI_ENTRY_VISIBILITY DUCKDB_EXTENSION_API bool DUCKDB_EXTENSION_GLUE(DUCKDB_EXTENSION_NAME,_init_c_api)(\
|
|
duckdb_extension_info info, struct duckdb_extension_access *access) {\
|
|
DUCKDB_EXTENSION_API_INIT(info, access, DUCKDB_EXTENSION_API_VERSION_STRING);\
|
|
return DUCKDB_EXTENSION_GLUE(DUCKDB_EXTENSION_NAME,_init_c_api_internal)(info, access);\
|
|
}\
|
|
DUCKDB_EXTENSION_EXTERN_C_GUARD_CLOSE static bool DUCKDB_EXTENSION_GLUE(DUCKDB_EXTENSION_NAME,_init_c_api_internal)
|
|
#endif
|
|
"""
|
|
|
|
with open(file, 'w+') as f:
|
|
f.write(duckdb_ext_h)
|
|
|
|
|
|
# Create duckdb_go_extension.h
|
|
def create_duckdb_go_ext_h(file, ext_api_version, function_groups, api_struct_definition, exclusion_set):
|
|
|
|
duckdb_ext_h = create_duckdb_ext_h_versioning(
|
|
ext_api_version,
|
|
function_groups,
|
|
api_struct_definition,
|
|
exclusion_set,
|
|
DUCKDB_HEADER_OUT_FILE_NAME,
|
|
DUCKDB_GO_H_HEADER,
|
|
with_member_invoker=True,
|
|
)
|
|
with open(file, 'w+') as f:
|
|
f.write(duckdb_ext_h)
|
|
|
|
|
|
# Create duckdb_extension_internal.hpp
|
|
def create_duckdb_ext_internal_h(ext_api_version, function_groups, function_map, ext_api_definitions, exclusion_set):
|
|
duckdb_ext_h = fetch_header_template_ext(DUCKDB_HEADER_OUT_FILE_NAME)
|
|
duckdb_ext_h += create_extension_api_struct(
|
|
function_groups,
|
|
function_map,
|
|
ext_api_definitions,
|
|
exclusion_set,
|
|
with_create_method=True,
|
|
create_method_name='CreateAPIv1',
|
|
)
|
|
duckdb_ext_h += create_version_defines(ext_api_version)
|
|
|
|
with open(DUCKDB_HEADER_EXT_INTERNAL_OUT_FILE, 'w+') as f:
|
|
f.write(duckdb_ext_h)
|
|
|
|
|
|
def get_extension_api_version(ext_api_definitions):
|
|
latest_version = ""
|
|
|
|
for version_entry in ext_api_definitions:
|
|
if version_entry['version'].startswith('v'):
|
|
latest_version = version_entry['version']
|
|
if version_entry['version'].startswith("unstable_"):
|
|
break
|
|
|
|
return latest_version
|
|
|
|
|
|
def create_struct_function_set(api_definitions):
|
|
result = set()
|
|
for api in api_definitions:
|
|
for entry in api['entries']:
|
|
if entry in result:
|
|
raise Exception(f"Duplicate entry found for function '{entry}'!")
|
|
result.add(entry)
|
|
return result
|
|
|
|
|
|
if __name__ == "__main__":
|
|
arg_parser = argparse.ArgumentParser(description='Generate the C API')
|
|
arg_parser.add_argument(
|
|
'-go', '--generate-go', type=bool, help='Emit the duckdb go extension header', required=False, default=False
|
|
)
|
|
args = arg_parser.parse_args()
|
|
|
|
# parse the api definition (which fields make it into the struct)
|
|
ext_api_definitions = parse_ext_api_definitions(EXT_API_DEFINITION_PATTERN)
|
|
|
|
# extract a set of the function names and the latest version of the api definition
|
|
ext_api_set = create_struct_function_set(ext_api_definitions)
|
|
ext_api_version = get_extension_api_version(ext_api_definitions)
|
|
|
|
function_groups, function_map = parse_capi_function_definitions()
|
|
function_map_size = len(function_map)
|
|
|
|
api_struct_function_count = sum([len(y['entries']) for y in ext_api_definitions])
|
|
ext_api_exclusion_set = parse_exclusion_list(function_map)
|
|
ext_api_exclusion_set_size = len(ext_api_exclusion_set)
|
|
|
|
print("Information")
|
|
print(f" * Current Extension C API Version: {ext_api_version}")
|
|
print(f" * Total functions: {function_map_size}")
|
|
print(f" * Functions in C API struct: {api_struct_function_count}")
|
|
print(f" * Functions in C API but excluded from struct: {ext_api_exclusion_set_size}")
|
|
|
|
print()
|
|
print("Generating headers")
|
|
|
|
print(f" * {DUCKDB_HEADER_OUT_FILE}")
|
|
create_duckdb_h(DUCKDB_HEADER_OUT_FILE, function_groups)
|
|
|
|
print(f" * {DUCKDB_HEADER_C_OUT_FILE}")
|
|
create_duckdb_c_ext_h(
|
|
DUCKDB_HEADER_C_OUT_FILE,
|
|
ext_api_version,
|
|
function_groups,
|
|
ext_api_definitions,
|
|
ext_api_exclusion_set,
|
|
)
|
|
|
|
print(f" * {DUCKDB_HEADER_EXT_INTERNAL_OUT_FILE}")
|
|
create_duckdb_ext_internal_h(
|
|
ext_api_version, function_groups, function_map, ext_api_definitions, ext_api_exclusion_set
|
|
)
|
|
print()
|
|
|
|
os.system(f"python3 scripts/format.py {DUCKDB_HEADER_OUT_FILE} --fix --noconfirm")
|
|
os.system(f"python3 scripts/format.py {DUCKDB_HEADER_C_OUT_FILE} --fix --noconfirm")
|
|
os.system(f"python3 scripts/format.py {DUCKDB_HEADER_EXT_INTERNAL_OUT_FILE} --fix --noconfirm")
|
|
|
|
print()
|
|
print("C API headers generated successfully!")
|
|
|
|
print()
|
|
if args.generate_go:
|
|
print(f" * {DUCKDB_HEADER_GO_OUT_FILE}")
|
|
create_duckdb_go_ext_h(
|
|
DUCKDB_HEADER_GO_OUT_FILE,
|
|
ext_api_version,
|
|
function_groups,
|
|
ext_api_definitions,
|
|
ext_api_exclusion_set,
|
|
)
|
|
os.system(f"python3 scripts/format.py {DUCKDB_HEADER_GO_OUT_FILE} --fix --noconfirm")
|
|
print()
|
|
print("Go C API headers generated successfully!")
|