name: ExtendedTests
on:
  workflow_dispatch:
  push:
    branches-ignore:
      - 'main'
      - 'feature'
      - 'v*.*-*'
    paths-ignore:
      - '**'
      - '!.github/workflows/ExtendedTests.yml'
  pull_request:
    types: [opened, reopened, ready_for_review]
    paths-ignore:
      - '**'
      - '!.github/workflows/ExtendedTests.yml'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
  cancel-in-progress: true

env:
  CCACHE_SAVE: ${{ github.repository != 'duckdb/duckdb' }}
  BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }}

jobs:
  regression-lto-benchmark-runner:
     name: Benchmark runner lto vs non-lto (OSX)
     runs-on: macos-latest
     env:
       CC: clang
       CXX: clang++
       GEN: ninja
       BUILD_BENCHMARK: 1
       CORE_EXTENSIONS: "tpch;tpcds;httpfs"

     steps:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0

     - uses: actions/setup-python@v5
       with:
         python-version: '3.12'

     - name: Install
       shell: bash
       run: brew install ninja llvm && pip install requests

     - name: Setup Ccache
       uses: hendrikmuhs/ccache-action@main
       with:
         key: ${{ github.job }}
         save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}

     - name: Build
       shell: bash
       run: |
         LTO=full make
         git clone --branch ${{ env.BASE_BRANCH }} https://github.com/duckdb/duckdb.git --depth=1
         cd duckdb
         make
         cd ..

     - name: Set up benchmarks
       shell: bash
       run: |
         cp -r benchmark duckdb/

     - name: Regression Test Micro
       if: always()
       shell: bash
       run: |
         python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/micro.csv  --threads 2

     - name: Regression Test TPCH
       if: always()
       shell: bash
       run: |
         python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpch.csv --threads 2
     - name: Regression Test TPCH-PARQUET
       if: always()
       shell: bash
       run: |
         python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpch_parquet.csv --threads 2


     - name: Regression Test TPCDS
       if: always()
       shell: bash
       run: |
         python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpcds.csv --threads 2


     - name: Regression Test H2OAI
       if: always()
       shell: bash
       run: |
         python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/h2oai.csv --threads 2

     - name: Regression Test IMDB
       if: always()
       shell: bash
       run: |
         python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/imdb.csv --threads 2

  regression-clang16-vs-clang14-benchmark-runner:
     name: Benchmark runner clang-16 vs clang-14 (OSX)
     runs-on: macos-latest
     env:
       CC: clang
       CXX: clang++
       GEN: ninja
       BUILD_BENCHMARK: 1
       CORE_EXTENSIONS: "tpch;tpcds;httpfs"

     steps:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0

     - uses: actions/setup-python@v5
       with:
         python-version: '3.12'

     - name: Install
       shell: bash
       run: brew install ninja llvm && pip install requests

     - name: Setup Ccache
       uses: hendrikmuhs/ccache-action@main
       with:
         key: ${{ github.job }}
         save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}

     - name: Build
       shell: bash
       run: |
         CMAKE_LLVM_PATH='/opt/homebrew/opt/llvm' make
         git clone --branch ${{ env.BASE_BRANCH }} https://github.com/duckdb/duckdb.git --depth=1
         cd duckdb
         make
         cd ..

     - name: Set up benchmarks
       shell: bash
       run: |
         cp -r benchmark duckdb/

     - name: Regression Test Micro
       if: always()
       shell: bash
       run: |
         python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/micro.csv  --threads 2

     - name: Regression Test TPCH
       if: always()
       shell: bash
       run: |
         python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpch.csv --threads 2

     - name: Regression Test TPCH-PARQUET
       if: always()
       shell: bash
       run: |
         python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpch_parquet.csv --threads 2

     - name: Regression Test TPCDS
       if: always()
       shell: bash
       run: |
         python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpcds.csv --threads 2

     - name: Regression Test H2OAI
       if: always()
       shell: bash
       run: |
         python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/h2oai.csv --threads 2

     - name: Regression Test IMDB
       if: always()
       shell: bash
       run: |
         python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/imdb.csv --threads 2

  regression-clang-benchmark-runner:
     name: Benchmark runner clang vs gcc
     runs-on: ubuntu-latest
     env:
       CC: gcc
       CXX: g++
       GEN: ninja
       BUILD_BENCHMARK: 1
       BUILD_JEMALLOC: 1
       CORE_EXTENSIONS: "tpch;tpcds;httpfs"

     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0

       - uses: actions/setup-python@v5
         with:
           python-version: '3.12'

       - name: Install
         shell: bash
         run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build llvm && pip install requests

       - name: Setup Ccache
         uses: hendrikmuhs/ccache-action@main
         with:
           key: ${{ github.job }}
           save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}

       - name: Build
         shell: bash
         run: |
           #### This should also be alternative way to instal llvm to a specific version
           # wget https://apt.llvm.org/llvm.sh
           # chmod +x llvm.sh
           # sudo ./llvm.sh 17
           #####
           CMAKE_LLVM_PATH='/usr/lib/llvm-14' make
           git clone --branch ${{ env.BASE_BRANCH }} https://github.com/duckdb/duckdb.git --depth=1
           cd duckdb
           make
           cd ..

       - name: Set up benchmarks
         shell: bash
         run: |
           cp -r benchmark duckdb/

       - name: Regression Test Micro
         if: always()
         shell: bash
         run: |
           python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/micro.csv --threads 2

       - name: Regression Test TPCH
         if: always()
         shell: bash
         run: |
           python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpch.csv --threads 2

       - name: Regression Test TPCH-PARQUET
         if: always()
         shell: bash
         run: |
           python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpch_parquet.csv --threads 2

       - name: Regression Test TPCDS
         if: always()
         shell: bash
         run: |
           python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpcds.csv --threads 2

       - name: Regression Test H2OAI
         if: always()
         shell: bash
         run: |
           python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/h2oai.csv --threads 2

       - name: Regression Test IMDB
         if: always()
         shell: bash
         run: |
           python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/imdb.csv --threads 2

  regression-flto-gcc-benchmark-runner:
     name: Benchmark runner gcc flto vs gcc
     runs-on: ubuntu-latest
     env:
       CC: gcc
       CXX: g++
       GEN: ninja
       BUILD_BENCHMARK: 1
       BUILD_JEMALLOC: 1
       CORE_EXTENSIONS: "tpch;tpcds;httpfs"

     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0

       - uses: actions/setup-python@v5
         with:
           python-version: '3.12'

       - name: Install
         shell: bash
         run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install requests

       - name: Setup Ccache
         uses: hendrikmuhs/ccache-action@main
         with:
           key: ${{ github.job }}
           save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}

       - name: Build
         shell: bash
         run: |
           LTO='full' make
           git clone --branch ${{ env.BASE_BRANCH }} https://github.com/duckdb/duckdb.git --depth=1
           cd duckdb
           make
           cd ..

       - name: Set up benchmarks
         shell: bash
         run: |
           cp -r benchmark duckdb/

       - name: Regression Test Micro
         if: always()
         shell: bash
         run: |
           python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/micro.csv --threads 2

       - name: Regression Test TPCH
         if: always()
         shell: bash
         run: |
           python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpch.csv --threads 2

       - name: Regression Test TPCH-PARQUET
         if: always()
         shell: bash
         run: |
           python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpch_parquet.csv --threads 2

       - name: Regression Test TPCDS
         if: always()
         shell: bash
         run: |
           python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpcds.csv --threads 2

       - name: Regression Test H2OAI
         if: always()
         shell: bash
         run: |
           python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/h2oai.csv --threads 2

       - name: Regression Test IMDB
         if: always()
         shell: bash
         run: |
           python scripts/regression/test_runner.py --nofail --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/imdb.csv --threads 2