366 lines
12 KiB
YAML
366 lines
12 KiB
YAML
#
|
|
# This workflow is responsible for building all DuckDB extensions
|
|
#
|
|
|
|
name: Extensions (all platforms)
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
override_git_describe:
|
|
type: string
|
|
git_ref:
|
|
type: string
|
|
skip_tests:
|
|
type: string
|
|
run_all:
|
|
type: string
|
|
workflow_dispatch:
|
|
inputs:
|
|
override_git_describe:
|
|
description: 'Version tag to override git describe. Use to produce binaries'
|
|
type: string
|
|
git_ref:
|
|
description: 'Set to override the DuckDB version, leave empty for current commit'
|
|
type: string
|
|
required: false
|
|
default: ''
|
|
extra_exclude_archs:
|
|
description: 'Inject more architectures to skip'
|
|
type: string
|
|
required: false
|
|
default: ''
|
|
skip_tests:
|
|
description: 'Set to true to skip all testing'
|
|
type: boolean
|
|
required: false
|
|
default: false
|
|
run_all:
|
|
type: string
|
|
required: false
|
|
default: 'true'
|
|
push:
|
|
branches-ignore:
|
|
- 'main'
|
|
- 'feature'
|
|
- 'v*.*-*'
|
|
paths-ignore:
|
|
- '**.md'
|
|
- 'tools/**'
|
|
- '!tools/shell/**'
|
|
- '.github/patches/duckdb-wasm/**'
|
|
- '.github/workflows/**'
|
|
- '!.github/workflows/Extensions.yml'
|
|
- '!.github/workflows/_extension_distribution.yml'
|
|
merge_group:
|
|
pull_request:
|
|
types: [opened, reopened, ready_for_review, converted_to_draft]
|
|
paths-ignore:
|
|
- '**.md'
|
|
- 'tools/**'
|
|
- '!tools/shell/**'
|
|
- '.github/patches/duckdb-wasm/**'
|
|
- '.github/workflows/**'
|
|
- '!.github/workflows/Extensions.yml'
|
|
|
|
concurrency:
|
|
group: extensions-${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}-${{ inputs.override_git_describe }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GH_TOKEN }}
|
|
|
|
jobs:
|
|
check-draft:
|
|
# We run all other jobs on PRs only if they are not draft PR
|
|
if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
|
|
runs-on: ubuntu-24.04
|
|
steps:
|
|
- name: Preliminary checks on CI
|
|
run: echo "Event name is ${{ github.event_name }}"
|
|
|
|
# This first step loads the various extension configs from the ~/.github/config directory storing them to drive the build jobs
|
|
load-extension-configs:
|
|
name: Load Extension Configs
|
|
runs-on: ubuntu-latest
|
|
needs: check-draft
|
|
outputs:
|
|
main_extensions_config: ${{ steps.set-main-extensions.outputs.extension_config }}
|
|
main_extensions_exclude_archs: ${{ steps.set-main-extensions.outputs.exclude_archs }}
|
|
rust_based_extensions_config: ${{ steps.set-rust-based-extensions.outputs.extension_config }}
|
|
rust_based_extensions_exclude_archs: ${{ steps.set-rust-based-extensions.outputs.exclude_archs }}
|
|
env:
|
|
# NOTE: on PRs we exclude some archs to speed things up
|
|
BASE_EXCLUDE_ARCHS: ${{ (github.event_name == 'pull_request' || inputs.run_all != 'true') && 'wasm_eh;wasm_threads;windows_amd64_mingw;osx_amd64;linux_arm64;linux_amd64_musl;' || '' }}
|
|
EXTRA_EXCLUDE_ARCHS: ${{ inputs.extra_exclude_archs }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
ref: ${{ inputs.git_ref }}
|
|
|
|
- id: set-main-extensions
|
|
name: Configure main extensions
|
|
env:
|
|
IN_TREE_CONFIG_FILE: .github/config/in_tree_extensions.cmake
|
|
OUT_OF_TREE_CONFIG_FILE: .github/config/out_of_tree_extensions.cmake
|
|
DEFAULT_EXCLUDE_ARCHS: ''
|
|
run: |
|
|
# Set config
|
|
echo exclude_archs="$DEFAULT_EXCLUDE_ARCHS;$BASE_EXCLUDE_ARCHS;$EXTRA_EXCLUDE_ARCHS" >> $GITHUB_OUTPUT
|
|
in_tree_extensions="`cat $IN_TREE_CONFIG_FILE`"
|
|
out_of_tree_extensions="`cat $OUT_OF_TREE_CONFIG_FILE`"
|
|
echo "extension_config<<EOF" >> $GITHUB_OUTPUT
|
|
echo "$in_tree_extensions" >> $GITHUB_OUTPUT
|
|
echo -e "\n" >> $GITHUB_OUTPUT
|
|
echo "$out_of_tree_extensions" >> $GITHUB_OUTPUT
|
|
echo "EOF" >> $GITHUB_OUTPUT
|
|
cat $GITHUB_OUTPUT
|
|
|
|
- id: set-rust-based-extensions
|
|
name: Configure Rust-based extensions
|
|
env:
|
|
CONFIG_FILE: .github/config/rust_based_extensions.cmake
|
|
DEFAULT_EXCLUDE_ARCHS: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw;linux_amd64_musl'
|
|
run: |
|
|
echo exclude_archs="$DEFAULT_EXCLUDE_ARCHS;$BASE_EXCLUDE_ARCHS;$EXTRA_EXCLUDE_ARCHS" >> $GITHUB_OUTPUT
|
|
rust_based_extensions="`cat .github/config/rust_based_extensions.cmake`"
|
|
echo "extension_config<<EOF" >> $GITHUB_OUTPUT
|
|
echo "$rust_based_extensions" >> $GITHUB_OUTPUT
|
|
echo "EOF" >> $GITHUB_OUTPUT
|
|
cat $GITHUB_OUTPUT
|
|
|
|
# Build the extensions from .github/config/in_tree_extensions.cmake
|
|
main-extensions:
|
|
name: Main Extensions
|
|
needs:
|
|
- load-extension-configs
|
|
uses: ./.github/workflows/_extension_distribution.yml
|
|
with:
|
|
artifact_prefix: main-extensions
|
|
exclude_archs: ${{ needs.load-extension-configs.outputs.main_extensions_exclude_archs }}
|
|
extension_config: ${{ needs.load-extension-configs.outputs.main_extensions_config }}
|
|
override_tag: ${{ inputs.override_git_describe }}
|
|
duckdb_ref: ${{ inputs.git_ref }}
|
|
skip_tests: ${{ inputs.skip_tests && true || false }}
|
|
save_cache: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
|
|
|
|
# Build the extensions from .github/config/rust_based_extensions.cmake
|
|
rust-based-extensions:
|
|
name: Rust-based Extensions
|
|
needs:
|
|
- load-extension-configs
|
|
uses: ./.github/workflows/_extension_distribution.yml
|
|
with:
|
|
exclude_archs: ${{ needs.load-extension-configs.outputs.rust_based_extensions_exclude_archs }}
|
|
artifact_prefix: rust-based-extensions
|
|
extension_config: ${{ needs.load-extension-configs.outputs.rust_based_extensions_config }}
|
|
extra_toolchains: 'rust'
|
|
override_tag: ${{ inputs.override_git_describe }}
|
|
duckdb_ref: ${{ inputs.git_ref }}
|
|
skip_tests: ${{ inputs.skip_tests && true || false }}
|
|
save_cache: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
|
|
|
|
# Merge all extensions into a single, versioned repository
|
|
create-extension-repository:
|
|
name: Create Extension Repository
|
|
runs-on: ubuntu-latest
|
|
needs:
|
|
- main-extensions
|
|
- rust-based-extensions
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- uses: actions/download-artifact@v4
|
|
name: Download main extensions
|
|
with:
|
|
pattern: main-extensions-${{ github.sha }}*
|
|
path: /tmp/repository_generation/main-extensions
|
|
|
|
- uses: actions/download-artifact@v4
|
|
name: Download rust-based extensions
|
|
with:
|
|
pattern: rust-based-extensions-${{ github.sha }}*
|
|
path: /tmp/repository_generation/rust-based-extensions
|
|
|
|
- name: Print all extensions
|
|
run: |
|
|
tree /tmp/repository_generation
|
|
|
|
- name: Merge into single repository
|
|
run: |
|
|
mkdir /tmp/merged_repository
|
|
cp -r /tmp/repository_generation/*/*/* /tmp/merged_repository
|
|
tree /tmp/merged_repository
|
|
|
|
- uses: actions/upload-artifact@v4
|
|
with:
|
|
if-no-files-found: error
|
|
name: extension-repository-${{ github.sha }}
|
|
path: |
|
|
/tmp/merged_repository/**/*.duckdb_extension*
|
|
|
|
upload-extensions:
|
|
name: Upload Extensions
|
|
runs-on: ubuntu-latest
|
|
needs:
|
|
- create-extension-repository
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- uses: actions/download-artifact@v4
|
|
with:
|
|
pattern: extension-repository-${{ github.sha }}
|
|
path: /tmp
|
|
|
|
- name: List extensions to deploy
|
|
shell: bash
|
|
run: |
|
|
tree /tmp/extension-repository-${{ github.sha }}
|
|
|
|
- name: Deploy extensions
|
|
shell: bash
|
|
env:
|
|
AWS_ENDPOINT_URL: ${{ secrets.DUCKDB_CORE_EXTENSION_S3_ENDPOINT }}
|
|
AWS_ACCESS_KEY_ID: ${{secrets.DUCKDB_CORE_EXTENSION_S3_ID}}
|
|
AWS_SECRET_ACCESS_KEY: ${{secrets.DUCKDB_CORE_EXTENSION_S3_SECRET}}
|
|
DUCKDB_DEPLOY_SCRIPT_MODE: for_real
|
|
DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }}
|
|
run: |
|
|
pip install awscli
|
|
./scripts/extension-upload-repository.sh /tmp/extension-repository-${{ github.sha }}
|
|
|
|
autoload-tests:
|
|
name: Extension Autoloading Tests
|
|
if: ${{ !inputs.skip_tests}}
|
|
runs-on: ubuntu-latest
|
|
needs: create-extension-repository
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
ref: ${{ inputs.git_ref }}
|
|
|
|
- name: Setup Build Environment
|
|
run: |
|
|
sudo apt-get update -y -qq
|
|
sudo apt-get install -y -qq ninja-build ccache
|
|
|
|
- name: Setup Ccache
|
|
uses: hendrikmuhs/ccache-action@main
|
|
with:
|
|
key: ${{ github.job }}
|
|
save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
|
|
|
|
- uses: actions/download-artifact@v4
|
|
with:
|
|
pattern: extension-repository-${{ github.sha }}
|
|
path: /tmp
|
|
|
|
- name: List extensions to test with
|
|
shell: bash
|
|
run: |
|
|
tree /tmp/extension-repository-${{ github.sha }}
|
|
|
|
- name: Build DuckDB
|
|
env:
|
|
GEN: ninja
|
|
CC: gcc
|
|
CXX: g++
|
|
EXTENSION_CONFIGS: './.github/config/rust_based_extensions.cmake;./.github/config/out_of_tree_extensions.cmake;./.github/config/in_tree_extensions.cmake'
|
|
EXTENSION_TESTS_ONLY: 1
|
|
ENABLE_EXTENSION_AUTOLOADING: 1
|
|
ENABLE_EXTENSION_AUTOINSTALL: 1
|
|
run: |
|
|
make release
|
|
|
|
- name: Run Tests
|
|
env:
|
|
LOCAL_EXTENSION_REPO: /tmp/extension-repository-${{ github.sha }}
|
|
run: |
|
|
./build/release/test/unittest --autoloading available --skip-compiled
|
|
|
|
check-load-install-extensions:
|
|
name: Checks extension entries
|
|
if: ${{ !inputs.skip_tests}}
|
|
runs-on: ubuntu-22.04
|
|
needs: create-extension-repository
|
|
env:
|
|
CC: gcc-10
|
|
CXX: g++-10
|
|
GEN: ninja
|
|
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
with:
|
|
fetch-depth: 0
|
|
ref: ${{ inputs.git_ref }}
|
|
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.9'
|
|
|
|
- name: Install
|
|
shell: bash
|
|
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
|
|
|
|
- name: Setup Ccache
|
|
uses: hendrikmuhs/ccache-action@main
|
|
with:
|
|
key: ${{ github.job }}
|
|
save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
|
|
|
|
- name: Build
|
|
shell: bash
|
|
env:
|
|
GENERATE_EXTENSION_ENTRIES: 1
|
|
LOCAL_EXTENSION_REPO: build/release/repository_other
|
|
run: |
|
|
make
|
|
|
|
- uses: actions/download-artifact@v4
|
|
name: Download extension repository artifact
|
|
with:
|
|
pattern: extension-repository-${{ github.sha }}
|
|
path: /tmp
|
|
|
|
- name: Copy over local extension repository
|
|
shell: bash
|
|
run: |
|
|
cp -r /tmp/extension-repository-${{ github.sha }} build/release/repository
|
|
tree build/release/repository
|
|
find build/release/repository -type f ! -path "build/release/repository/*/linux_amd64/*" -delete
|
|
tree build/release/repository
|
|
|
|
- name: Check if extension_entries.hpp is up to date
|
|
shell: bash
|
|
env:
|
|
EXTENSION_CONFIGS: '.github/config/in_tree_extensions.cmake;.github/config/out_of_tree_extensions.cmake'
|
|
run: |
|
|
make extension_configuration
|
|
python scripts/generate_extensions_function.py
|
|
pip install "black>=24"
|
|
pip install cmake-format
|
|
pip install "clang_format==11.0.1"
|
|
make format-fix
|
|
|
|
- uses: actions/upload-artifact@v4
|
|
with:
|
|
name: extension_entries.hpp
|
|
path: |
|
|
src/include/duckdb/main/extension_entries.hpp
|
|
|
|
- name: Check for any difference
|
|
run: |
|
|
git diff --exit-code src/include/duckdb/main/extension_entries.hpp && echo "No differences found"
|
|
|
|
- name: Explainer
|
|
if: failure()
|
|
run: |
|
|
echo "There are differences in src/include/duckdb/main/extension_entries.hpp"
|
|
echo "Check the uploaded extension_entries.hpp (in the workflow Summary), and check that in instead of src/include/duckdb/main/extension_entries.hpp" |