name: Main
on:
  workflow_dispatch:
  repository_dispatch:
  push:
    branches-ignore:
      - 'main'
      - 'feature'
      - 'v*.*-*'
    paths-ignore:
      - '**.md'
      - 'tools/**'
      - '!tools/shell/**'
      - '.github/patches/duckdb-wasm/**'
      - '.github/workflows/**'
      - '!.github/workflows/Main.yml'
      - '.github/config/extensions/*.cmake'
      - '.github/patches/extensions/**/*.patch'
      - '!.github/patches/extensions/fts/*.patch' # fts used in some jobs
      - '!.github/config/extensions/fts.cmake'
  merge_group:
  pull_request:
    types: [opened, reopened, ready_for_review, converted_to_draft]
    paths-ignore:
      - '**.md'
      - 'tools/**'
      - '!tools/shell/**'
      - '.github/patches/duckdb-wasm/**'
      - '.github/workflows/**'
      - '!.github/workflows/Main.yml'
      - '.github/config/extensions/*.cmake'
      - '.github/patches/extensions/**/*.patch'
      - '!.github/patches/extensions/fts/*.patch' # fts used in some jobs
      - '!.github/config/extensions/fts.cmake'


concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
  cancel-in-progress: true

env:
  GH_TOKEN: ${{ secrets.GH_TOKEN }}

jobs:
 check-draft:
    # We run all other jobs on PRs only if they are not draft PR
    if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
    runs-on: ubuntu-24.04
    steps:
      - name: Preliminary checks on CI 
        run: echo "Event name is ${{ github.event_name }}"

 linux-debug:
    name: Linux Debug
    # This tests release build while enabling slow verifiers (masked by #ifdef DEBUG) and sanitizers
    needs: check-draft
    runs-on: ubuntu-22.04
    env:
      CC: gcc-10
      CXX: g++-10
      TREAT_WARNINGS_AS_ERRORS: 1
      GEN: ninja
      CRASH_ON_ASSERT: 1
      CMAKE_CXX_FLAGS: '-DDEBUG'
      FORCE_ASSERT: 1

    steps:
    - uses: actions/checkout@v4

    - id: describe_step
      run: echo "git_describe=$(git describe --tags --long)" >> "$GITHUB_OUTPUT"

    - name: Install
      shell: bash
      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@main
      with:
        key: ${{ github.job }}
        save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}

    - name: Build
      shell: bash
      run:  make release

    - name: Output version info
      shell: bash
      run: ./build/release/duckdb -c "PRAGMA version;"

    - name: Set DUCKDB_INSTALL_LIB for ADBC tests
      shell: bash
      run: echo "DUCKDB_INSTALL_LIB=$(find `pwd` -name "libduck*.so" | head -n 1)" >> $GITHUB_ENV

    - name: Test DUCKDB_INSTALL_LIB variable
      run: echo $DUCKDB_INSTALL_LIB

    - name: Test
      shell: bash
      run: |
        python3 scripts/run_tests_one_by_one.py build/release/test/unittest --tests-per-invocation 100

 linux-release:
    name: Linux Release (full suite)
    needs: check-draft
    runs-on: ubuntu-24.04
    env:
      GEN: ninja
      BUILD_JEMALLOC: 1
      CORE_EXTENSIONS: "icu;tpch;tpcds;fts;json;inet"
      DISABLE_SANITIZER: 1

    steps:
    - uses: actions/checkout@v3

    - name: Install
      shell: bash
      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@main
      with:
        key: ${{ github.job }}
        save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}

    - name: Build
      shell: bash
      run: make release

    - name: Test
      shell: bash
      run: make allunit

 no-string-inline:
    name: No String Inline / Destroy Unpinned Blocks
    runs-on: ubuntu-24.04
    needs: linux-configs
    env:
      GEN: ninja
      CORE_EXTENSIONS: "icu;parquet;tpch;tpcds;fts;json;inet"
      DISABLE_STRING_INLINE: 1
      DESTROY_UNPINNED_BLOCKS: 1
      ALTERNATIVE_VERIFY: 1
      DISABLE_POINTER_SALT: 1
      LSAN_OPTIONS: suppressions=${{ github.workspace }}/.sanitizer-leak-suppressions.txt
      DUCKDB_TEST_DESCRIPTION: 'Compiled with ALTERNATIVE_VERIFY=1 DISABLE_STRING_INLINE=1 DESTROY_UNPINNED_BLOCKS=1 DISABLE_POINTER_SALT=1. Use require no_alternative_verify to skip.'

    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Install
        shell: bash
        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build

      - name: Setup Ccache
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}
          save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}

      - name: Build
        shell: bash
        run: make relassert

      - name: Test
        shell: bash
        run: build/relassert/test/unittest

 vector-sizes:
    name: Vector Sizes
    runs-on: ubuntu-22.04
    needs: linux-configs
    env:
      CC: gcc-10
      CXX: g++-10
      GEN: ninja
      DUCKDB_TEST_DESCRIPTION: 'Compiled with STANDARD_VECTOR_SIZE=2. Use require vector_size 2048 to skip tests.'

    steps:
      - name: Clean up the disc space
        shell: bash
        run: |
          echo "Disk usage before clean up:"
          df -h
          rm -rf /opt/hostedtoolcache/CodeQL Java* Pypy Ruby go node
          echo "Disk usage after clean up:"
          df -h

      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Install
        shell: bash
        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build

      - name: Setup Ccache
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}
          save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}

      - name: Build
        shell: bash
        run: STANDARD_VECTOR_SIZE=2 make reldebug

      - name: Test
        shell: bash
        run: |
          python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest --no-exit --time_execution

 valgrind:
    name: Valgrind
    if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
    runs-on: ubuntu-24.04
    needs: linux-configs
    env:
      CC: clang
      CXX: clang++
      DISABLE_SANITIZER: 1
      BUILD_JEMALLOC: 1
      CORE_EXTENSIONS: 'icu;json;parquet;tpch'
      GEN: ninja

    steps:
    - uses: actions/checkout@v4

    - name: Install
      shell: bash
      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build valgrind clang

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@main
      with:
        key: ${{ github.job }}
        save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}

    - name: Build
      shell: bash
      run: make relassert

    - name: Output version info
      shell: bash
      run: ./build/relassert/duckdb -c "PRAGMA version;"

    - name: Test
      shell: bash
      run: valgrind ./build/relassert/test/unittest test/sql/tpch/tpch_sf001.test_slow

 threadsan:
    name: Thread Sanitizer
    needs: linux-configs
    runs-on: ubuntu-24.04
    env:
      CC: clang
      CXX: clang++
      GEN: ninja
      BUILD_JEMALLOC: 1
      CORE_EXTENSIONS: "icu;tpch;tpcds;fts;json;inet"
      TSAN_OPTIONS: suppressions=${{ github.workspace }}/.sanitizer-thread-suppressions.txt
      DUCKDB_TEST_DESCRIPTION: 'Tests run with thread sanitizer.'

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - name: Install
      shell: bash
      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build clang

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@main
      with:
        key: ${{ github.job }}
        save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}

    - name: Build
      shell: bash
      run: THREADSAN=1 make reldebug

    - name: Test
      shell: bash
      run: |
          python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest --no-exit --timeout 600
          python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "[intraquery]" --no-exit --timeout 600
          python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "[interquery]" --no-exit --timeout 1800
          python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "[interquery]" --no-exit --timeout 1800 --force-storage
          python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "[interquery]" --no-exit --timeout 1800 --force-storage --force-reload
          python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "[detailed_profiler]" --no-exit --timeout 600
          python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest test/sql/tpch/tpch_sf01.test_slow --no-exit --timeout 600

 amalgamation-tests:
    name: Amalgamation Tests
    runs-on: ubuntu-22.04
    needs: check-draft
    env:
      CC: clang
      CXX: clang++

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
        ref: ${{ github.ref }}

    - uses: actions/setup-python@v5
      with:
        python-version: '3.12'

    - name: Install LLVM and Clang
      uses: KyleMayes/install-llvm-action@v1
      with:
        version: "14.0"

    - name: Generate Amalgamation
      shell: bash
      run:  |
          python scripts/amalgamation.py --extended
          clang++ -std=c++17 -Isrc/amalgamation src/amalgamation/duckdb.cpp -emit-llvm -S -O0

 force-blocking-sink-source:
    name: Forcing async Sinks/Sources
    runs-on: ubuntu-24.04
    needs: check-draft
    env:
      GEN: ninja
      CORE_EXTENSIONS: "icu;parquet;tpch;tpcds;fts;json;inet"
      FORCE_ASYNC_SINK_SOURCE: 1

    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Install
        shell: bash
        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build

      - name: Setup Ccache
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}
          save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}

      - name: Build
        shell: bash
        run: make relassert

      - name: Test
        shell: bash
        run: python3 scripts/run_tests_one_by_one.py build/relassert/test/unittest --no-exit --timeout 600



 # TODO: Bring back BLOCK_VERIFICATION: 1, and consider bringing back fts
 # TODO: DEBUG_STACKTRACE: 1 + reldebug ?
 linux-configs:
    name: Tests a release build with different configurations
    runs-on: ubuntu-24.04
    needs: check-draft
    env:
      BASE_BRANCH: ${{ github.base_ref || 'main' }}

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - uses: actions/setup-python@v5
      with:
        python-version: '3.12'

    - name: Install Ninja
      shell: bash
      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@main
      with:
        key: ${{ github.job }}
        save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}

    - name: Build
      id: build
      shell: bash
      env:
        CORE_EXTENSIONS: "json;parquet;icu;tpch;tpcds"
        GEN: ninja
      run: make

    - name: test/configs/encryption.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/encryption.json

    - name: test/configs/force_storage.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/force_storage.json

    - name: test/configs/force_storage_restart.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/force_storage_restart.json

    - name: test/configs/latest_storage.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/latest_storage.json

    - name: test/configs/verify_fetch_row.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/verify_fetch_row.json

    - name: test/configs/wal_verification.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/wal_verification.json

    - name: test/configs/prefetch_all_parquet_files.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/prefetch_all_parquet_files.json

    - name: test/configs/no_local_filesystem.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/no_local_filesystem.json

    - name: test/configs/block_size_16kB.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/block_size_16kB.json

    - name: test/configs/latest_storage_block_size_16kB.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/latest_storage_block_size_16kB.json

    - name: test/configs/enable_verification.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/enable_verification.json

    - name: Test dictionary_expression
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --verify-vector dictionary_expression --skip-compiled

    - name: Test dictionary_operator
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --verify-vector dictionary_operator --skip-compiled

    - name: Test constant_operator
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --verify-vector constant_operator --skip-compiled

    - name: Test sequence_operator
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --verify-vector sequence_operator --skip-compiled

    - name: Test nested_shuffle
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --verify-vector nested_shuffle --skip-compiled

    - name: Test variant_vector
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/variant_vector.json

    - name: Test variant_vector
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/compressed_in_memory.json

    - name: Test block prefetching
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/prefetch_all_storage.json

    - name: Test peg_parser
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/peg_parser.json
    - name: Forwards compatibility tests
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        python3 scripts/test_storage_compatibility.py --versions "1.2.1|1.3.2" --new-unittest build/release/test/unittest