110 lines
4.0 KiB
Bash
Executable File
110 lines
4.0 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# This script deploys the extension binaries that are currently deployed to the nightly bucket to the main bucket
|
|
|
|
# WARNING: don't use this script if you don't know exactly what you're doing. To deploy a binary:
|
|
# - Run the script with ./extension-upload-from-nightly.sh <extension_name> <duckdb_version> (<nightly_commit>)
|
|
# - CHECK the output of the dry run thoroughly
|
|
# - If successful, set the DUCKDB_DEPLOY_SCRIPT_MODE env variable to the correct value
|
|
# - run the script again now deploying for real
|
|
# - check the output
|
|
# - unset the DUCKDB_DEPLOY_SCRIPT_MODE env var
|
|
|
|
if [ -z "$1" ] || [ -z "$2" ]; then
|
|
echo "Usage: ./extension-upload-from-nightly.sh <extension_name> <duckdb_version> (<nightly_commit>)"
|
|
exit 1
|
|
fi
|
|
|
|
if [ -z "$3" ]; then
|
|
BASE_NIGHTLY_DIR="$2"
|
|
else
|
|
BASE_NIGHTLY_DIR="$1/$3/$2"
|
|
fi
|
|
|
|
# CONFIG
|
|
FROM_BUCKET=duckdb-extensions-nightly
|
|
TO_BUCKET=duckdb-core-extensions
|
|
CLOUDFRONT_DISTRIBUTION_ID=E2Z28NDMI4PVXP
|
|
|
|
### COPY THE FILES
|
|
## REAL_RUN is to be used to move non-Wasm extensions
|
|
REAL_RUN="aws s3 cp s3://$FROM_BUCKET/$BASE_NIGHTLY_DIR s3://$TO_BUCKET/$2 --recursive --exclude '*' --include '*/$1.duckdb_extension.gz' --acl public-read --region us-east-2"
|
|
DRY_RUN="$REAL_RUN --dryrun"
|
|
## REAL_RUN_WASM is to be used to move Wasm extensions to new style path (no extra duckdb-wasm)
|
|
REAL_RUN_WASM="aws s3 cp s3://$FROM_BUCKET/$BASE_NIGHTLY_DIR s3://$TO_BUCKET/$2 --recursive --exclude '*' --include '*/$1.duckdb_extension.wasm' --acl public-read --content-encoding br --content-type='application/wasm' --region us-east-2"
|
|
DRY_RUN_WASM="$REAL_RUN_WASM --dryrun"
|
|
|
|
if [ "$DUCKDB_DEPLOY_SCRIPT_MODE" == "for_real" ]; then
|
|
echo "DEPLOYING"
|
|
echo "> FROM: $FROM_BUCKET"
|
|
echo "> TO : $TO_BUCKET"
|
|
echo "> AWS CLI deploy: "
|
|
eval "$REAL_RUN"
|
|
eval "$REAL_RUN_WASM"
|
|
else
|
|
echo "DEPLOYING (DRY RUN)"
|
|
echo "> FROM: $FROM_BUCKET"
|
|
echo "> TO : $TO_BUCKET"
|
|
echo "> AWS CLI Dry run: "
|
|
eval "$DRY_RUN"
|
|
eval "$DRY_RUN_WASM"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
### INVALIDATE THE CLOUDFRONT CACHE AND CLOUDFLARE
|
|
# For double checking we are invalidating the correct domain
|
|
CLOUDFRONT_ORIGINS=`aws cloudfront get-distribution --id $CLOUDFRONT_DISTRIBUTION_ID --query 'Distribution.DistributionConfig.Origins.Items[*].DomainName' --output text`
|
|
|
|
# Parse the dry run output
|
|
output=$(eval "$DRY_RUN" && eval "$DRY_RUN_WASM" && eval "$DRY_RUN_WASM_OLD_STYLE")
|
|
s3_paths=()
|
|
while IFS= read -r line; do
|
|
if [[ $line == *"copy:"* ]]; then
|
|
s3_path=$(echo $line | grep -o 's3://[^ ]*' | awk 'NR%2==0' | awk -F "s3://$TO_BUCKET" '{print $2}' | cut -d' ' -f1)
|
|
s3_paths+=("$s3_path")
|
|
fi
|
|
done <<< "$output"
|
|
|
|
if [ "$DUCKDB_DEPLOY_SCRIPT_MODE" == "for_real" ]; then
|
|
echo "CLOUDFRONT INVALIDATION"
|
|
echo "> Total files: ${#s3_paths[@]}"
|
|
echo "> Domain: $CLOUDFRONT_ORIGINS"
|
|
for path in "${s3_paths[@]}"; do
|
|
aws cloudfront create-invalidation --distribution-id "$CLOUDFRONT_DISTRIBUTION_ID" --paths "$path"
|
|
done
|
|
else
|
|
echo "INVALIDATION (DRY RUN)"
|
|
echo "> Total files: ${#s3_paths[@]}"
|
|
echo "> Domain: $CLOUDFRONT_ORIGINS"
|
|
echo "> Paths:"
|
|
for path in "${s3_paths[@]}"; do
|
|
echo " $path"
|
|
done
|
|
fi
|
|
|
|
echo ""
|
|
|
|
if [ ! -z "$CLOUDFLARE_CACHE_PURGE_TOKEN" ]; then
|
|
if [ "$DUCKDB_DEPLOY_SCRIPT_MODE" == "for_real" ]; then
|
|
echo "CLOUDFLARE INVALIDATION"
|
|
echo "> Total files: ${#s3_paths[@]}"
|
|
for path in "${s3_paths[@]}"; do
|
|
curl --request POST --url https://api.cloudflare.com/client/v4/zones/84f631c38b77d4631b561207f2477332/purge_cache --header 'Content-Type: application/json' --header "Authorization: Bearer $CLOUDFLARE_CACHE_PURGE_TOKEN" --data "{\"files\": [\"http://extensions.duckdb.org$path\"]}"
|
|
echo ""
|
|
done
|
|
else
|
|
echo "CLOUDFLARE INVALIDATION (DRY RUN)"
|
|
echo "> Total files: ${#s3_paths[@]}"
|
|
echo "> Domain: $CLOUDFRONT_ORIGINS"
|
|
echo "> Paths:"
|
|
for path in "${s3_paths[@]}"; do
|
|
echo " http://extensions.duckdb.org$path"
|
|
done
|
|
fi
|
|
else
|
|
echo "##########################################"
|
|
echo "WARNING! CLOUDFLARE INVALIDATION DISABLED!"
|
|
echo "##########################################"
|
|
fi
|