should be it
This commit is contained in:
177
external/duckdb/scripts/run_extension_medata_tests.sh
vendored
Executable file
177
external/duckdb/scripts/run_extension_medata_tests.sh
vendored
Executable file
@@ -0,0 +1,177 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Generates a bunch of directories to be used for testing extension updating related behaviour used in `test/extension/update_extensions_ci.test`
|
||||
|
||||
# Please consider your energy footprint by only running this script with ccache.
|
||||
# note that subsequent runs used cached artifacts, use `make clean` or rm -rf build/debug to clean
|
||||
|
||||
set -x
|
||||
set -e
|
||||
|
||||
DUCKDB_BUILD_DIR="./build/debug"
|
||||
|
||||
TEST_DIR="./build/extension_metadata_test_data"
|
||||
TEST_DIR_COPY="./build/extension_metadata_test_data_copy"
|
||||
|
||||
### Directories to use
|
||||
# Used as the extension installation directory for DuckDB
|
||||
export LOCAL_EXTENSION_DIR="$TEST_DIR/extension_dir"
|
||||
# Repository for testing successfully updating extensions
|
||||
export LOCAL_EXTENSION_REPO_UPDATED="$TEST_DIR/repository"
|
||||
# Repository for testing incorrect platform
|
||||
export LOCAL_EXTENSION_REPO_INCORRECT_PLATFORM="$TEST_DIR/repository_incorrect_platform"
|
||||
# Repository for testing incorrect version
|
||||
export LOCAL_EXTENSION_REPO_INCORRECT_DUCKDB_VERSION="$TEST_DIR/repository_incorrect_version"
|
||||
# Repository where both platform and version mismatch
|
||||
export LOCAL_EXTENSION_REPO_VERSION_AND_PLATFORM_INCORRECT="$TEST_DIR/repository_incorrect_version_and_platform"
|
||||
# Directory containing the extensions for direct installing
|
||||
export DIRECT_INSTALL_DIR="$TEST_DIR/direct_install"
|
||||
|
||||
# Extension dir with a malformed info file for an extension
|
||||
export LOCAL_EXTENSION_DIR_MALFORMED_INFO="$TEST_DIR/extension_dir_malformed_info"
|
||||
# Extension dir with a metadata install version that mismatches the files metadata
|
||||
export LOCAL_EXTENSION_DIR_INFO_INCORRECT_VERSION="$TEST_DIR/extension_dir_malformed_info_incorrect_version"
|
||||
|
||||
if [ -d "$TEST_DIR_COPY" ]; then
|
||||
# REUSE PREVIOUSLY GENERATED DATA
|
||||
rm -r $TEST_DIR
|
||||
cp -R $TEST_DIR_COPY $TEST_DIR
|
||||
else
|
||||
# GENERATE FRESH DATA
|
||||
mkdir -p $TEST_DIR
|
||||
mkdir -p $DIRECT_INSTALL_DIR
|
||||
mkdir -p $LOCAL_EXTENSION_DIR
|
||||
mkdir -p $LOCAL_EXTENSION_REPO_UPDATED
|
||||
mkdir -p $LOCAL_EXTENSION_REPO_INCORRECT_PLATFORM
|
||||
mkdir -p $LOCAL_EXTENSION_REPO_INCORRECT_DUCKDB_VERSION
|
||||
|
||||
#################################################
|
||||
### First repo: successfully updating extensions.
|
||||
#################################################
|
||||
# Set extension config
|
||||
cat > $TEST_DIR/extension_config_before.cmake <<EOL
|
||||
duckdb_extension_load(json DONT_LINK EXTENSION_VERSION v0.0.1)
|
||||
duckdb_extension_load(tpch DONT_LINK EXTENSION_VERSION v0.0.1)
|
||||
duckdb_extension_load(tpcds DONT_LINK EXTENSION_VERSION v0.0.1)
|
||||
duckdb_extension_load(icu DONT_LINK EXTENSION_VERSION v0.0.1)
|
||||
EOL
|
||||
|
||||
# Build the extensions using the first config
|
||||
LOCAL_EXTENSION_REPO=$LOCAL_EXTENSION_REPO_UPDATED EXTENSION_CONFIGS=$TEST_DIR/extension_config_before.cmake make debug
|
||||
|
||||
# Set the version and platform now that we have a build
|
||||
DUCKDB_VERSION=`$DUCKDB_BUILD_DIR/duckdb -csv -noheader -c 'select source_id from pragma_version()'`
|
||||
DUCKDB_PLATFORM=`cat $DUCKDB_BUILD_DIR/duckdb_platform_out`
|
||||
|
||||
# Install the extension from the initial config
|
||||
$DUCKDB_BUILD_DIR/duckdb -unsigned -c "set extension_directory='$LOCAL_EXTENSION_DIR'; set custom_extension_repository='$LOCAL_EXTENSION_REPO_UPDATED'; install tpch; install json; INSTALL icu;"
|
||||
|
||||
# Delete the info file from the icu extension
|
||||
rm $LOCAL_EXTENSION_DIR/$DUCKDB_VERSION/$DUCKDB_PLATFORM/icu.duckdb_extension.info
|
||||
|
||||
# Install tpcds directly
|
||||
cp $DUCKDB_BUILD_DIR/extension/tpcds/tpcds.duckdb_extension $DIRECT_INSTALL_DIR/tpcds.duckdb_extension
|
||||
$DUCKDB_BUILD_DIR/duckdb -unsigned -c "set extension_directory='$LOCAL_EXTENSION_DIR'; install '$DIRECT_INSTALL_DIR/tpcds.duckdb_extension';"
|
||||
|
||||
# Set updated extension config where we update the tpch extension but not the json extension
|
||||
cat > $TEST_DIR/extension_config_after.cmake <<EOL
|
||||
duckdb_extension_load(json DONT_LINK EXTENSION_VERSION v0.0.1)
|
||||
duckdb_extension_load(tpch DONT_LINK EXTENSION_VERSION v0.0.2)
|
||||
duckdb_extension_load(icu DONT_LINK EXTENSION_VERSION v0.0.2)
|
||||
EOL
|
||||
|
||||
# Build the extensions using the second config
|
||||
LOCAL_EXTENSION_REPO=$LOCAL_EXTENSION_REPO_UPDATED EXTENSION_CONFIGS=$TEST_DIR/extension_config_after.cmake BUILD_EXTENSIONS_ONLY=1 make debug
|
||||
|
||||
# For good measure, we also gzip one of the files in the repo to ensure we can do both gzipped and non gzipped
|
||||
gzip -1 $LOCAL_EXTENSION_REPO_UPDATED/$DUCKDB_VERSION/$DUCKDB_PLATFORM/icu.duckdb_extension
|
||||
|
||||
##########################################
|
||||
### Second repo: Incorrect DuckDB platform
|
||||
##########################################
|
||||
rm -rf $DUCKDB_BUILD_DIR
|
||||
# Set extension config
|
||||
cat > $TEST_DIR/extension_config_incorrect_platform.cmake <<EOL
|
||||
duckdb_extension_load(json DONT_LINK EXTENSION_VERSION v0.0.3)
|
||||
EOL
|
||||
|
||||
# Build the extensions using the incorrect platform
|
||||
DUCKDB_PLATFORM=test_platform EXTENSION_CONFIGS=$TEST_DIR/extension_config_incorrect_platform.cmake BUILD_EXTENSIONS_ONLY=1 make debug
|
||||
|
||||
cp $DUCKDB_BUILD_DIR/extension/json/json.duckdb_extension $DIRECT_INSTALL_DIR/json_incorrect_platform.duckdb_extension
|
||||
|
||||
########################################
|
||||
### Third repo: Incorrect DuckDB version
|
||||
########################################
|
||||
rm -rf $DUCKDB_BUILD_DIR
|
||||
# Set extension config
|
||||
cat > $TEST_DIR/extension_config_incorrect_version.cmake <<EOL
|
||||
duckdb_extension_load(json DONT_LINK EXTENSION_VERSION v0.0.4)
|
||||
EOL
|
||||
|
||||
# Build the extensions using the incorrect platform
|
||||
DUCKDB_EXPLICIT_VERSION=v1337 EXTENSION_CONFIGS=$TEST_DIR/extension_config_before.cmake BUILD_EXTENSIONS_ONLY=1 make debug
|
||||
|
||||
cp $DUCKDB_BUILD_DIR/extension/json/json.duckdb_extension $DIRECT_INSTALL_DIR/json_incorrect_version.duckdb_extension
|
||||
|
||||
####################################################
|
||||
### Fourth repo: Both platform and version incorrect
|
||||
####################################################
|
||||
rm -rf $DUCKDB_BUILD_DIR
|
||||
# Set extension config
|
||||
cat > $TEST_DIR/extension_config_incorrect_version.cmake <<EOL
|
||||
duckdb_extension_load(json DONT_LINK EXTENSION_VERSION v0.0.4)
|
||||
EOL
|
||||
|
||||
# Build the extensions using the incorrect platform
|
||||
DUCKDB_PLATFORM=test_platform DUCKDB_EXPLICIT_VERSION=v1337 EXTENSION_CONFIGS=$TEST_DIR/extension_config_before.cmake BUILD_EXTENSIONS_ONLY=1 make debug
|
||||
|
||||
cp $DUCKDB_BUILD_DIR/extension/json/json.duckdb_extension $DIRECT_INSTALL_DIR/json_incorrect_version_and_platform.duckdb_extension
|
||||
|
||||
# Note that we set the "double wrong" extension to have the proper name, so we can actually load it during testing with
|
||||
# SET allow_extensions_metadata_mismatch=true;
|
||||
cp $DUCKDB_BUILD_DIR/extension/json/json.duckdb_extension $DIRECT_INSTALL_DIR/json.duckdb_extension
|
||||
|
||||
###########################
|
||||
### Prepare malformed repos/dirs
|
||||
###########################
|
||||
# Build clean duckdb
|
||||
rm -rf $DUCKDB_BUILD_DIR
|
||||
make debug
|
||||
|
||||
# Use duckdb to install the extensions into the repositories (note that we are doing a trick here by setting the extension_directory to the local repo dir)
|
||||
$DUCKDB_BUILD_DIR/duckdb -unsigned -c "set allow_extensions_metadata_mismatch=true; set extension_directory='$LOCAL_EXTENSION_REPO_INCORRECT_PLATFORM'; install '$DIRECT_INSTALL_DIR/json_incorrect_platform.duckdb_extension'"
|
||||
$DUCKDB_BUILD_DIR/duckdb -unsigned -c "set allow_extensions_metadata_mismatch=true; set extension_directory='$LOCAL_EXTENSION_REPO_INCORRECT_DUCKDB_VERSION'; install '$DIRECT_INSTALL_DIR/json_incorrect_version.duckdb_extension'"
|
||||
$DUCKDB_BUILD_DIR/duckdb -unsigned -c "set allow_extensions_metadata_mismatch=true; set extension_directory='$LOCAL_EXTENSION_REPO_VERSION_AND_PLATFORM_INCORRECT'; install '$DIRECT_INSTALL_DIR/json_incorrect_version_and_platform.duckdb_extension'"
|
||||
|
||||
# Create dir with malformed info file
|
||||
$DUCKDB_BUILD_DIR/duckdb -unsigned -c "set extension_directory='$LOCAL_EXTENSION_DIR_MALFORMED_INFO'; install '$DIRECT_INSTALL_DIR/tpcds.duckdb_extension';"
|
||||
echo blablablab > $LOCAL_EXTENSION_DIR_MALFORMED_INFO/$DUCKDB_VERSION/$DUCKDB_PLATFORM/tpcds.duckdb_extension.info
|
||||
|
||||
# Create dir with malformed info file: we install a new version from LOCAL_EXTENSION_REPO_UPDATED but preserve the old info file
|
||||
$DUCKDB_BUILD_DIR/duckdb -unsigned -c "set extension_directory='$LOCAL_EXTENSION_DIR_INFO_INCORRECT_VERSION'; install 'tpch' from '$LOCAL_EXTENSION_REPO_UPDATED'"
|
||||
cp $LOCAL_EXTENSION_DIR/$DUCKDB_VERSION/$DUCKDB_PLATFORM/tpch.duckdb_extension.info $LOCAL_EXTENSION_DIR_INFO_INCORRECT_VERSION/$DUCKDB_VERSION/$DUCKDB_PLATFORM/tpch.duckdb_extension.info
|
||||
|
||||
###################################################################
|
||||
### Allow using copy instead of regenerating test data on every run
|
||||
###################################################################
|
||||
cp -R $TEST_DIR $TEST_DIR_COPY
|
||||
fi
|
||||
|
||||
###########################
|
||||
### Set version and platform
|
||||
###########################
|
||||
DUCKDB_VERSION=`$DUCKDB_BUILD_DIR/duckdb -csv -noheader -c 'select source_id from pragma_version()'`
|
||||
DUCKDB_PLATFORM=`cat $DUCKDB_BUILD_DIR/duckdb_platform_out`
|
||||
|
||||
###########################
|
||||
### Populate the minio repositories
|
||||
###########################
|
||||
AWS_DEFAULT_REGION=eu-west-1 AWS_ACCESS_KEY_ID=minio_duckdb_user AWS_SECRET_ACCESS_KEY=minio_duckdb_user_password aws --endpoint-url http://duckdb-minio.com:9000 s3 sync $LOCAL_EXTENSION_REPO_UPDATED s3://test-bucket-public/ci-test-repo
|
||||
export REMOTE_EXTENSION_REPO_UPDATED=http://duckdb-minio.com:9000/test-bucket-public/ci-test-repo
|
||||
export REMOTE_EXTENSION_REPO_DIRECT_PATH=http://duckdb-minio.com:9000/test-bucket-public/ci-test-repo/$DUCKDB_VERSION/$DUCKDB_PLATFORM
|
||||
|
||||
################
|
||||
### Run test
|
||||
################
|
||||
RUN_EXTENSION_UPDATE_TEST=1 $DUCKDB_BUILD_DIR/test/unittest test/extension/update_extensions_ci.test
|
||||
Reference in New Issue
Block a user