feat: Introducing pg_lakehouse

.dockerignore

@@ -0,0 +1,9 @@
+**/*.csv
+**/*.csv.gz
+**/*.json
+**/*.parquet
+**/*.tbl
+**/*.tsv
+**/*.tsv.gz
+**/*.txt
+**/*.zip

.github/workflows/benchmark-pg_lakehouse.yml

@@ -0,0 +1,52 @@
+# workflows/benchmark-pg_lakehouse.yml
+#
+# Benchmark pg_lakehouse
+# Benchmark ParadeDB's pg_lakehouse performance on a nightly basis. This workflow can also be triggered
+# manually to benchmark other systems on one-off basis, to compare against ParadeDB.
+
+name: Benchmark pg_lakehouse
+
+on:
+  schedule:
+    - cron: "1 0 * * 1,2,3,4,5" # Run once per day on weekdays (days of the week 1-5) at 00:01 UTC
+  pull_request:
+    branches:
+      - dev
+      - staging
+      - main
+    paths:
+      - "docker/Dockerfile"
+      - "pg_lakehouse/Cargo.toml"
+      - "pg_lakehouse/benchmarks/**"
+      - "shared/Cargo.toml"
+      - ".github/workflows/benchmark-pg_lakehouse.yml"
+  workflow_dispatch:
+
+concurrency:
+  group: benchmark-pg_lakehouse-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  benchmark-pg_lakehouse:
+    name: Benchmark pg_lakehouse on ${{ matrix.name }}
+    runs-on: ubuntu-latest-m # Size: 4-cores · 16 GB RAM · 150 GB SSD
+    strategy:
+      matrix:
+        include:
+          - name: ClickBench (Parquet, single)
+            flags: -w single
+          - name: ClickBench (Parquet, partitioned)
+            flags: -w partitioned
+
+    steps:
+      - name: Checkout Git Repository
+        uses: actions/checkout@v4
+
+      - name: Run Official ${{ matrix.name }} Benchmark
+        working-directory: pg_lakehouse/benchmarks/
+        run: ./benchmark.sh -t local ${{ matrix.flags }}
+
+      - name: Notify Slack on Failure
+        if: failure() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
+        run: |
+          curl -X POST -H 'Content-type: application/json' --data '{"text":"Benchmark pg_lakehouse on ${{ matrix.name }} h workflow failed -- investigate immediately!"}' ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/benchmark-pg_search.yml

@@ -18,6 +18,8 @@ on:
       - "docker/Dockerfile"
       - "pg_search/Cargo.toml"
       - "pg_search/benchmarks/**"
+      - "shared/Cargo.toml"
+      - "tokenizers/Cargo.toml"
       - ".github/workflows/benchmark-pg_search.yml"
   workflow_dispatch:
     inputs:

.github/workflows/publish-pg_lakehouse.yml

@@ -0,0 +1,134 @@
+# workflows/publish-pg_lakehouse.yml
+#
+# Publish pg_lakehouse
+# Build and publish the pg_lakehouse extension as a .deb to GitHub Releases.
+
+name: Publish pg_lakehouse
+
+on:
+  push:
+    tags:
+      - "v*"
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "The version to set for the pg_lakehouse release. This publishes the latest commit of the chosen branch and uploads it to the pre-existing GitHub Release of the provided version."
+        required: true
+        default: ""
+
+concurrency:
+  group: publish-pg_lakehouse-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  publish-pg_lakehouse:
+    name: Publish pg_lakehouse for PostgreSQL ${{ matrix.pg_version }} on ${{ matrix.arch }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+          - runner: ubuntu-latest
+            pg_version: 15
+            arch: amd64
+          - runner: ubicloud-standard-8-ubuntu-2204-arm
+            pg_version: 15
+            arch: arm64
+          - runner: ubuntu-latest
+            pg_version: 16
+            arch: amd64
+          - runner: ubicloud-standard-8-ubuntu-2204-arm
+            pg_version: 16
+            arch: arm64
+
+    steps:
+      - name: Checkout Git Repository
+        uses: actions/checkout@v4
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Retrieve Ubuntu & GitHub Tag Versions
+        id: version
+        run: |
+          # If no workflow_dispatch version is provided, we use workflow tag trigger version
+          if [ -z "${{ github.event.inputs.version }}" ]; then
+            echo "version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
+          else
+            echo "version=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT
+          fi
+          echo "ubuntu_version=$(lsb_release -rs | sed 's/\.//')" >> $GITHUB_OUTPUT
+
+      - name: Install & Configure Supported PostgreSQL Version
+        run: |
+          wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+          sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+          sudo apt-get update && sudo apt-get install -y postgresql-${{ matrix.pg_version }} postgresql-server-dev-${{ matrix.pg_version }} debhelper devscripts dput gnupg
+          sudo chown -R $(whoami) /usr/share/postgresql/${{ matrix.pg_version }}/extension/ /usr/lib/postgresql/${{ matrix.pg_version }}/lib/
+          echo "/usr/lib/postgresql/${{ matrix.pg_version }}/bin" >> $GITHUB_PATH
+
+      # Note: We require a different version of pgrx (0.11.3) due to version compatibility
+      # with the supabase/wrappers project on which pg_lakehouse is based
+      - name: Install pgrx
+        run: cargo install -j $(nproc) --locked cargo-pgrx --version 0.11.3
+
+      - name: Initialize pgrx for Current PostgreSQL Version
+        working-directory: pg_lakehouse/
+        run: cargo pgrx init --pg${{ matrix.pg_version }}=/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config
+
+      # TODO: Add telemetry to pg_lakehouse
+      - name: Package pg_lakehouse Extension with pgrx
+        working-directory: pg_lakehouse/
+        run: cargo pgrx package --features telemetry
+
+      - name: Create .deb Package
+        run: |
+          # Create installable package
+          mkdir archive
+          cp `find target/release -type f -name "pg_lakehouse*"` archive
+          package_dir=pg_lakehouse-${{ steps.version.outputs.version }}-pg${{ matrix.pg_version }}-${{ matrix.arch }}-ubuntu${{ steps.version.outputs.ubuntu_version }}
+
+          # Copy files into directory structure
+          mkdir -p ${package_dir}/usr/lib/postgresql/lib
+          mkdir -p ${package_dir}/var/lib/postgresql/extension
+          cp archive/*.so ${package_dir}/usr/lib/postgresql/lib
+          cp archive/*.control ${package_dir}/var/lib/postgresql/extension
+          cp archive/*.sql ${package_dir}/var/lib/postgresql/extension
+
+          # Symlinks to copy files into directory structure
+          mkdir -p ${package_dir}/usr/lib/postgresql/${{ matrix.pg_version }}/lib
+          mkdir -p ${package_dir}/usr/share/postgresql/${{ matrix.pg_version}}/extension
+          cp archive/*.so ${package_dir}/usr/lib/postgresql/${{ matrix.pg_version }}/lib
+          cp archive/*.control ${package_dir}/usr/share/postgresql/${{ matrix.pg_version }}/extension
+          cp archive/*.sql ${package_dir}/usr/share/postgresql/${{ matrix.pg_version }}/extension
+
+          # Create control file (package name cannot have underscore)
+          mkdir -p ${package_dir}/DEBIAN
+          touch ${package_dir}/DEBIAN/control
+          deb_version=${{ steps.version.outputs.version }}
+          CONTROL_FILE="${package_dir}/DEBIAN/control"
+          echo 'Package: pg-lakehouse' >> $CONTROL_FILE
+          echo 'Version:' ${deb_version} >> $CONTROL_FILE
+          echo 'Section: database' >> $CONTROL_FILE
+          echo 'Priority: optional' >> $CONTROL_FILE
+          echo 'Architecture: ${{ matrix.arch }}' >> $CONTROL_FILE
+          echo 'Depends: postgresql-${{ matrix.pg_version }}' >> $CONTROL_FILE
+          echo 'Maintainer: ParadeDB <support@paradedb.com>' >> $CONTROL_FILE
+          echo 'Description: An analytical query engine for Postgres' >> $CONTROL_FILE
+
+          # Create .deb package
+          sudo chown -R root:root ${package_dir}
+          sudo chmod -R 00755 ${package_dir}
+          sudo dpkg-deb --build --root-owner-group ${package_dir}
+
+      # We retrieve the GitHub release for the specific release version
+      - name: Retrieve GitHub Release Upload URL
+        id: upload_url
+        run: echo UPLOAD_URL=$(curl --silent https://api.github.com/repos/${{ github.repository }}/releases/tags/v${{ steps.version.outputs.version }} | jq .upload_url --raw-output) >> $GITHUB_OUTPUT
+
+      - name: Upload pg_lakehouse .deb to GitHub Release
+        uses: shogo82148/actions-upload-release-asset@v1
+        with:
+          github_token: ${{ secrets.GHA_CREATE_RELEASE_PAT }}
+          upload_url: ${{ steps.upload_url.outputs.upload_url }}
+          asset_path: ./pg_lakehouse-${{ steps.version.outputs.version }}-pg${{ matrix.pg_version }}-${{ matrix.arch }}-ubuntu${{ steps.version.outputs.ubuntu_version }}.deb
+          asset_name: pg_lakehouse-v${{ steps.version.outputs.version }}-pg${{ matrix.pg_version }}-${{ matrix.arch }}-ubuntu${{ steps.version.outputs.ubuntu_version }}.deb

.github/workflows/test-paradedb.yml

@@ -14,7 +14,10 @@ on:
       - dev
     paths:
       - "docker/**"
+      - "pg_lakehouse/src/**"
       - "pg_search/src/**"
+      - "shared/**"
+      - "tokenizers/**"
       - ".github/workflows/test-paradedb.yml"
   workflow_dispatch:
 

.github/workflows/test-pg_lakehouse.yml

@@ -0,0 +1,155 @@
+# workflows/test-pg_lakehouse.yml
+#
+# Test pg_lakehouse
+# Run unit and integration tests for the pg_lakehouse extension.
+
+name: Test pg_lakehouse
+
+on:
+  pull_request:
+    branches:
+      - dev
+      - staging
+      - main
+    paths:
+      - "pg_lakehouse/**"
+      - "!pg_lakehouse/README.md"
+      - "shared/**"
+      - ".github/workflows/test-pg_lakehouse.yml"
+  push:
+    branches:
+      - dev # Run CI on dev. This is important to fill the GitHub Actions cache in a way that pull requests can see it
+  workflow_dispatch:
+    inputs:
+      test_upgrade_version:
+        description: "Upcoming pg_lakehouse version to test upgrading against"
+        required: false
+        default: ""
+
+concurrency:
+  group: test-pg_lakehouse-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test-pg_lakehouse:
+    name: Test pg_lakehouse on PostgreSQL ${{ matrix.pg_version }} for ${{ matrix.arch }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+          - runner: ubuntu-latest
+            pg_version: 14
+            arch: amd64
+          - runner: ubuntu-latest
+            pg_version: 15
+            arch: amd64
+          - runner: ubuntu-latest
+            pg_version: 16
+            arch: amd64
+    env:
+      default_pg_version: 16
+
+    steps:
+      # For the Rust cache to get filled, we need to run the CI on the dev branch after every merge. This only
+      # needs to happen once, so we skip the workflow for all but one of the matrix jobs in that case.
+      - name: Check if Skipping
+        id: check_skip
+        run: |
+          if [[ "${{ github.event_name }}" = "push" && "${{ matrix.pg_version }}" != "${{ env.default_pg_version }}" ]]; then
+            echo "This is a push event to fill Rust cache. Skipping this job."
+            echo "skip_remaining_steps=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Checkout Git Repository
+        if: steps.check_skip.outputs.skip_remaining_steps != 'true'
+        uses: actions/checkout@v4
+
+      # This checks that the version in Cargo.toml is incremented to the next release. We only run it
+      # on PRs to main, which are our release promotion PRs.
+      - name: Check version in Cargo.toml
+        if: steps.check_skip.outputs.skip_remaining_steps != 'true' && github.base_ref == 'main'
+        working-directory: pg_lakehouse/
+        run: |
+          CARGO_VERSION=$(grep "^version" Cargo.toml | head -1 | awk -F '"' '{print $2}')
+          RELEASE_VERSION="${{ vars.VERSION_MAJOR }}.${{ vars.VERSION_MINOR }}.${{ vars.VERSION_PATCH }}"
+          if [ "$CARGO_VERSION" != "$RELEASE_VERSION" ]; then
+            echo "Version in Cargo.toml ($CARGO_VERSION) does not match upcoming release version ($RELEASE_VERSION), did you forget to increment it?"
+            exit 1
+          fi
+          echo "Version check passed!"
+
+      - name: Install Rust
+        if: steps.check_skip.outputs.skip_remaining_steps != 'true'
+        uses: dtolnay/rust-toolchain@stable
+
+      # Caches from base branches are available to PRs, but not across unrelated branches, so we only
+      # save the cache on the 'dev' branch, but load it on all branches.
+      - name: Install Rust Cache
+        if: steps.check_skip.outputs.skip_remaining_steps != 'true'
+        uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: "v1"
+          shared-key: ${{ runner.os }}-rust-cache-pg_lakehouse-${{ HashFiles('Cargo.lock') }}
+          cache-targets: true
+          cache-on-failure: true
+          cache-all-crates: true
+          save-if: ${{ github.ref == 'refs/heads/dev' }}
+
+      - name: Install & Configure Supported PostgreSQL Version
+        if: steps.check_skip.outputs.skip_remaining_steps != 'true'
+        run: |
+          wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+          sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+          sudo apt-get update && sudo apt-get install -y postgresql-${{ matrix.pg_version }} postgresql-server-dev-${{ matrix.pg_version }}
+          sudo chown -R $(whoami) /usr/share/postgresql/${{ matrix.pg_version }}/extension/ /usr/lib/postgresql/${{ matrix.pg_version }}/lib/ /var/lib/postgresql/${{ matrix.pg_version }}/ /usr/lib/postgresql/${{ matrix.pg_version }}/bin/
+          echo "/usr/lib/postgresql/${{ matrix.pg_version }}/bin" >> $GITHUB_PATH
+
+      # Note: We require a different version of pgrx (0.11.3) due to version compatibility
+      # with the supabase/wrappers project on which pg_lakehouse is based
+      - name: Install pgrx, grcov & llvm-tools-preview
+        if: steps.check_skip.outputs.skip_remaining_steps != 'true'
+        run: |
+          cargo install -j $(nproc) --locked cargo-pgrx --version 0.11.3
+          cargo install -j $(nproc) --locked grcov
+          rustup component add llvm-tools-preview
+
+      # Our Rust Cache action caches the entire ~/target directory, which includes the pg_lakehouse extension. Since we
+      # only want to restore dependencies (to speed up the build) but not the extension itself (so the tests are representative), we
+      # clean any cached pg_lakehouse & dependencies build artifacts before running the tests
+      - name: Clean Cached Build Artifacts
+        if: steps.check_skip.outputs.skip_remaining_steps != 'true' && matrix.pg_version == env.default_pg_version
+        working-directory: pg_lakehouse/
+        run: cargo clean
+
+      # The integration tests also test upgrading the extension when passed the '-u' flag (only on promotion PRs)
+      - name: Run pg_lakehouse Cargo Test Suite
+        if: steps.check_skip.outputs.skip_remaining_steps != 'true'
+        working-directory: pg_lakehouse/
+        env:
+          LLVM_PROFILE_FILE: target/coverage/pg_lakehouse-%p-%m.profraw
+          RUST_BACKTRACE: full
+        run: |
+          # Add the code coverage flags to the config.toml file
+          echo -e "\n# Enable code coverage on Linux only, for CI builds\n[target.'cfg(target_os=\"linux\")']\nrustflags = [\"-Cinstrument-coverage\"]" >> ../.cargo/config.toml
+          mkdir -p ../target/coverage ../target/coverage-report
+
+          if [[ "${{ github.base_ref }}" == "main" && "${{ matrix.pg_version }}" == "${{ env.default_pg_version }}" ]]; then
+            # Retrieve the version to test upgrading to
+            if [[ "${{ github.event.inputs.test_upgrade_version }}" == "" ]]; then
+              echo "New version not provided, assuming the next release is the GitHub Actions Variable version..."
+              NEW_VERSION=${{ vars.VERSION_MAJOR }}.${{ vars.VERSION_MINOR }}.${{ vars.VERSION_PATCH }}
+            else
+              echo "Using the provided new version (likely a manual major or minor version increment)..."
+              NEW_VERSION="${{ github.event.inputs.test_upgrade_version }}"
+            fi
+            echo "Version to test upgrading to: $NEW_VERSION"
+            ./runtests.sh -p sequential -v "${{ matrix.pg_version }}" -u "$NEW_VERSION" -a "${{ matrix.arch }}"
+          else
+            ./runtests.sh -p sequential -v "${{ matrix.pg_version }}"
+          fi
+
+      # On promotion PRs, we test packaging the extension with production features enabled
+      - name: Test Packaging pg_lakehouse
+        if: steps.check_skip.outputs.skip_remaining_steps != 'true' && github.base_ref == 'main'
+        working-directory: pg_lakehouse/
+        run: cargo clean && cargo pgrx package --features telemetry

.github/workflows/test-pg_search.yml

@@ -14,6 +14,7 @@ on:
     paths:
       - "pg_search/**"
       - "tokenizers/**"
+      - "shared/**"
       - "!pg_search/README.md"
       - ".github/workflows/test-pg_search.yml"
   push:

.gitignore

@@ -5,7 +5,6 @@
 .DS_Store
 
 # Cargo
-Cargo.lock
 target/
 
 # Docker