From bcbb074df47327617a317c61c5b5b3335b361670 Mon Sep 17 00:00:00 2001 From: psteinroe Date: Sun, 31 Aug 2025 16:00:10 +0200 Subject: [PATCH 1/4] chore: offline builds with submodules --- .gitmodules | 3 + crates/pgt_query/build.rs | 115 +++++++++++++++++----------- crates/pgt_query/vendor/libpg_query | 1 + 3 files changed, 73 insertions(+), 46 deletions(-) create mode 160000 crates/pgt_query/vendor/libpg_query diff --git a/.gitmodules b/.gitmodules index 9b62ce889..2d44388d4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,6 @@ path = lib/tree_sitter_sql/tree-sitter-sql url = https://github.com/DerekStride/tree-sitter-sql branch = gh-pages +[submodule "crates/pgt_query/vendor/libpg_query"] + path = crates/pgt_query/vendor/libpg_query + url = https://github.com/pganalyze/libpg_query.git diff --git a/crates/pgt_query/build.rs b/crates/pgt_query/build.rs index 292b3af28..b07b84f4a 100644 --- a/crates/pgt_query/build.rs +++ b/crates/pgt_query/build.rs @@ -1,6 +1,3 @@ -#![cfg_attr(feature = "clippy", feature(plugin))] -#![cfg_attr(feature = "clippy", plugin(clippy))] - use fs_extra::dir::CopyOptions; use glob::glob; use std::env; @@ -8,12 +5,12 @@ use std::path::PathBuf; use std::process::Command; static LIBRARY_NAME: &str = "pg_query"; -static LIBPG_QUERY_REPO: &str = "https://github.com/pganalyze/libpg_query.git"; + fn get_libpg_query_tag() -> &'static str { #[cfg(feature = "postgres-15")] - return "15-5.3.0"; + return "15-4.2.4"; #[cfg(feature = "postgres-16")] - return "16-6.1.0"; + return "16-5.2.0"; #[cfg(feature = "postgres-17")] return "17-6.1.0"; } @@ -21,60 +18,86 @@ fn get_libpg_query_tag() -> &'static str { fn main() -> Result<(), Box> { let libpg_query_tag = get_libpg_query_tag(); let out_dir = PathBuf::from(env::var("OUT_DIR")?); - let vendor_dir = out_dir.join("vendor"); - let libpg_query_dir = vendor_dir.join("libpg_query").join(libpg_query_tag); - let stamp_file = libpg_query_dir.join(".stamp"); + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); + let libpg_query_submodule = manifest_dir.join("vendor").join("libpg_query"); - let src_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).join("src"); + let src_dir = manifest_dir.join("src"); let target = env::var("TARGET").unwrap(); let is_emscripten = target.contains("emscripten"); - // Configure cargo through stdout println!("cargo:rustc-link-search=native={}", out_dir.display()); println!("cargo:rustc-link-lib=static={LIBRARY_NAME}"); - // Clone libpg_query if not already present - if !stamp_file.exists() { - println!("cargo:warning=Cloning libpg_query {}", libpg_query_tag); + if !libpg_query_submodule.join(".git").exists() && !libpg_query_submodule.join("src").exists() { + return Err( + "libpg_query submodule not found. Please run: git submodule update --init --recursive" + .into(), + ); + } - // Create vendor directory - std::fs::create_dir_all(&vendor_dir)?; + // check if we need to checkout a different tag + let current_head = Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(&libpg_query_submodule) + .output()?; + + let tag_commit = Command::new("git") + .args(["rev-list", "-n", "1", libpg_query_tag]) + .current_dir(&libpg_query_submodule) + .output(); + + let needs_checkout = match tag_commit { + Ok(output) => { + let current = String::from_utf8_lossy(¤t_head.stdout); + let target = String::from_utf8_lossy(&output.stdout); + current.trim() != target.trim() + } + Err(_) => { + // tag not found locally, try fetching + let status = Command::new("git") + .args(["fetch", "--tags"]) + .current_dir(&libpg_query_submodule) + .status()?; + + if !status.success() { + return Err("Failed to fetch tags from libpg_query".into()); + } + true + } + }; - // Clone the repository with partial clone for faster download + if needs_checkout { + // checkout the correct tag for the selected postgresql version + println!( + "cargo:warning=Checking out libpg_query tag: {}", + libpg_query_tag + ); let status = Command::new("git") - .args([ - "clone", - "--filter=blob:none", - "--depth", - "1", - "--branch", - libpg_query_tag, - LIBPG_QUERY_REPO, - libpg_query_dir.to_str().unwrap(), - ]) + .args(["checkout", libpg_query_tag]) + .current_dir(&libpg_query_submodule) .status()?; if !status.success() { - return Err("Failed to clone libpg_query".into()); + return Err(format!("Failed to checkout libpg_query tag: {}", libpg_query_tag).into()); } - - // Create stamp file - std::fs::File::create(&stamp_file)?; } - // Tell cargo to rerun if the stamp file is deleted - println!("cargo:rerun-if-changed={}", stamp_file.display()); + // tell cargo to rerun if the submodule changes + println!( + "cargo:rerun-if-changed={}", + libpg_query_submodule.join("src").display() + ); - // Copy necessary files to OUT_DIR for compilation + // copy necessary files to out_dir for compilation let out_header_path = out_dir.join(LIBRARY_NAME).with_extension("h"); let out_protobuf_path = out_dir.join("protobuf"); let source_paths = vec![ - libpg_query_dir.join(LIBRARY_NAME).with_extension("h"), - libpg_query_dir.join("Makefile"), - libpg_query_dir.join("src"), - libpg_query_dir.join("protobuf"), - libpg_query_dir.join("vendor"), + libpg_query_submodule.join(LIBRARY_NAME).with_extension("h"), + libpg_query_submodule.join("Makefile"), + libpg_query_submodule.join("src"), + libpg_query_submodule.join("protobuf"), + libpg_query_submodule.join("vendor"), ]; let copy_options = CopyOptions { @@ -84,17 +107,17 @@ fn main() -> Result<(), Box> { fs_extra::copy_items(&source_paths, &out_dir, ©_options)?; - // Compile the C library. + // compile the c library. let mut build = cc::Build::new(); - // Configure for Emscripten if needed + // configure for emscripten if needed if is_emscripten { - // Use emcc as the compiler instead of gcc/clang + // use emcc as the compiler instead of gcc/clang build.compiler("emcc"); - // Use emar as the archiver instead of ar + // use emar as the archiver instead of ar build.archiver("emar"); - // Note: We don't add WASM-specific flags here as this creates a static library - // The final linking flags should be added when building the final WASM module + // note: we don't add wasm-specific flags here as this creates a static library + // the final linking flags should be added when building the final wasm module } build @@ -115,7 +138,7 @@ fn main() -> Result<(), Box> { .include(out_dir.join("./vendor")) .include(out_dir.join("./src/postgres/include")) .include(out_dir.join("./src/include")) - .warnings(false); // Avoid unnecessary warnings, as they are already considered as part of libpg_query development + .warnings(false); // avoid unnecessary warnings, as they are already considered as part of libpg_query development if env::var("PROFILE").unwrap() == "debug" || env::var("DEBUG").unwrap() == "1" { build.define("USE_ASSERT_CHECKING", None); } diff --git a/crates/pgt_query/vendor/libpg_query b/crates/pgt_query/vendor/libpg_query new file mode 160000 index 000000000..1c1a32ed2 --- /dev/null +++ b/crates/pgt_query/vendor/libpg_query @@ -0,0 +1 @@ +Subproject commit 1c1a32ed2f4c7799830d50bf4cb159222aafec48 From eb2c84cdf30d43d81e4cc8aba84883507b704058 Mon Sep 17 00:00:00 2001 From: psteinroe Date: Sun, 31 Aug 2025 17:00:23 +0200 Subject: [PATCH 2/4] progress --- .gitmodules | 2 ++ crates/pgt_query/build.rs | 23 ++++++++++++----------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.gitmodules b/.gitmodules index 2d44388d4..98c271783 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,3 +5,5 @@ [submodule "crates/pgt_query/vendor/libpg_query"] path = crates/pgt_query/vendor/libpg_query url = https://github.com/pganalyze/libpg_query.git + fetchRecurseSubmodules = true + shallow = false diff --git a/crates/pgt_query/build.rs b/crates/pgt_query/build.rs index b07b84f4a..a87c36d88 100644 --- a/crates/pgt_query/build.rs +++ b/crates/pgt_query/build.rs @@ -30,7 +30,7 @@ fn main() -> Result<(), Box> { if !libpg_query_submodule.join(".git").exists() && !libpg_query_submodule.join("src").exists() { return Err( - "libpg_query submodule not found. Please run: git submodule update --init --recursive" + "libpg_query submodule not found. Please run: git submodule update --init --recursive && cd crates/pgt_query/vendor/libpg_query && git fetch --tags" .into(), ); } @@ -53,16 +53,17 @@ fn main() -> Result<(), Box> { current.trim() != target.trim() } Err(_) => { - // tag not found locally, try fetching - let status = Command::new("git") - .args(["fetch", "--tags"]) - .current_dir(&libpg_query_submodule) - .status()?; - - if !status.success() { - return Err("Failed to fetch tags from libpg_query".into()); - } - true + // tag not found locally + return Err(format!( + "Tag {} not found in libpg_query submodule. Please run:\n\ + cd {} && git fetch --tags && git checkout {} && git checkout {} && git checkout {}", + libpg_query_tag, + libpg_query_submodule.display(), + "15-4.2.4", + "16-5.2.0", + "17-6.1.0" + ) + .into()); } }; From a30e84b83fc873db7e3d887bddafb0a36194ac24 Mon Sep 17 00:00:00 2001 From: psteinroe Date: Sun, 31 Aug 2025 20:42:25 +0200 Subject: [PATCH 3/4] progress --- .gitmodules | 1 + Cargo.lock | 14 ++-- crates/pgt_lexer_codegen/build.rs | 27 ++++---- .../17-6.1.0 => vendor/17-latest}/kwlist.h | 0 crates/pgt_query/Cargo.toml | 5 +- crates/pgt_query/build.rs | 64 ++----------------- crates/pgt_query/vendor/libpg_query | 2 +- crates/pgt_query_macros/build.rs | 27 ++------ .../17-latest/pg_query.proto} | 0 9 files changed, 32 insertions(+), 108 deletions(-) rename crates/pgt_lexer_codegen/{postgres/17-6.1.0 => vendor/17-latest}/kwlist.h (100%) rename crates/pgt_query_macros/{postgres/17-6.1.0.proto => vendor/17-latest/pg_query.proto} (100%) diff --git a/.gitmodules b/.gitmodules index 98c271783..e104f8b02 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,4 @@ url = https://github.com/pganalyze/libpg_query.git fetchRecurseSubmodules = true shallow = false + branch = 17-latest diff --git a/Cargo.lock b/Cargo.lock index be7eacb75..7011a88da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3863,9 +3863,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.28" +version = "0.23.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" +checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" dependencies = [ "log", "once_cell", @@ -3887,9 +3887,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.3" +version = "0.103.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" +checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc" dependencies = [ "ring", "rustls-pki-types", @@ -5325,14 +5325,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.1", + "webpki-roots 1.0.2", ] [[package]] name = "webpki-roots" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8782dd5a41a24eed3a4f40b606249b3e236ca61adf1f25ea4d45c73de122b502" +checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" dependencies = [ "rustls-pki-types", ] diff --git a/crates/pgt_lexer_codegen/build.rs b/crates/pgt_lexer_codegen/build.rs index 70c9635d4..fc15b7749 100644 --- a/crates/pgt_lexer_codegen/build.rs +++ b/crates/pgt_lexer_codegen/build.rs @@ -3,39 +3,34 @@ use std::fs; use std::io::Write; use std::path::PathBuf; -// TODO make this selectable via feature flags -static LIBPG_QUERY_TAG: &str = "17-6.1.0"; +static LIBPG_QUERY_TAG: &str = "17-latest"; -/// Downloads the `kwlist.h` file from the specified version of `libpg_query` fn main() -> Result<(), Box> { - let version = LIBPG_QUERY_TAG.to_string(); - - // Check for the postgres header file in the source tree first let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); - let headers_dir = manifest_dir.join("postgres").join(&version); - let kwlist_path = headers_dir.join("kwlist.h"); + let vendor_dir = manifest_dir.join("vendor").join(LIBPG_QUERY_TAG); + let kwlist_path = vendor_dir.join("kwlist.h"); - // Only download if the file doesn't exist + // Download kwlist.h if not already present in source directory if !kwlist_path.exists() { println!( - "cargo:warning=Downloading kwlist.h for libpg_query {}", - version + "cargo:warning=Downloading kwlist.h for libpg_query {} to source directory", + LIBPG_QUERY_TAG ); - fs::create_dir_all(&headers_dir)?; + fs::create_dir_all(&vendor_dir)?; - let proto_url = format!( + let kwlist_url = format!( "https://raw.githubusercontent.com/pganalyze/libpg_query/{}/src/postgres/include/parser/kwlist.h", - version + LIBPG_QUERY_TAG ); - let response = ureq::get(&proto_url).call()?; + let response = ureq::get(&kwlist_url).call()?; let content = response.into_string()?; let mut file = fs::File::create(&kwlist_path)?; file.write_all(content.as_bytes())?; - println!("cargo:warning=Successfully downloaded kwlist.h"); + println!("cargo:warning=Successfully downloaded kwlist.h to source"); } println!( diff --git a/crates/pgt_lexer_codegen/postgres/17-6.1.0/kwlist.h b/crates/pgt_lexer_codegen/vendor/17-latest/kwlist.h similarity index 100% rename from crates/pgt_lexer_codegen/postgres/17-6.1.0/kwlist.h rename to crates/pgt_lexer_codegen/vendor/17-latest/kwlist.h diff --git a/crates/pgt_query/Cargo.toml b/crates/pgt_query/Cargo.toml index 881b1b800..8fdbe6ca1 100644 --- a/crates/pgt_query/Cargo.toml +++ b/crates/pgt_query/Cargo.toml @@ -18,10 +18,7 @@ pgt_query_macros = { workspace = true } [features] -default = ["postgres-17"] -postgres-15 = [] -postgres-16 = [] -postgres-17 = [] +default = [] [build-dependencies] bindgen = "0.72.0" diff --git a/crates/pgt_query/build.rs b/crates/pgt_query/build.rs index a87c36d88..6570746bc 100644 --- a/crates/pgt_query/build.rs +++ b/crates/pgt_query/build.rs @@ -6,17 +6,7 @@ use std::process::Command; static LIBRARY_NAME: &str = "pg_query"; -fn get_libpg_query_tag() -> &'static str { - #[cfg(feature = "postgres-15")] - return "15-4.2.4"; - #[cfg(feature = "postgres-16")] - return "16-5.2.0"; - #[cfg(feature = "postgres-17")] - return "17-6.1.0"; -} - fn main() -> Result<(), Box> { - let libpg_query_tag = get_libpg_query_tag(); let out_dir = PathBuf::from(env::var("OUT_DIR")?); let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); let libpg_query_submodule = manifest_dir.join("vendor").join("libpg_query"); @@ -28,62 +18,15 @@ fn main() -> Result<(), Box> { println!("cargo:rustc-link-search=native={}", out_dir.display()); println!("cargo:rustc-link-lib=static={LIBRARY_NAME}"); + // Check if submodule exists if !libpg_query_submodule.join(".git").exists() && !libpg_query_submodule.join("src").exists() { return Err( - "libpg_query submodule not found. Please run: git submodule update --init --recursive && cd crates/pgt_query/vendor/libpg_query && git fetch --tags" + "libpg_query submodule not found. Please run: git submodule update --init --recursive" .into(), ); } - // check if we need to checkout a different tag - let current_head = Command::new("git") - .args(["rev-parse", "HEAD"]) - .current_dir(&libpg_query_submodule) - .output()?; - - let tag_commit = Command::new("git") - .args(["rev-list", "-n", "1", libpg_query_tag]) - .current_dir(&libpg_query_submodule) - .output(); - - let needs_checkout = match tag_commit { - Ok(output) => { - let current = String::from_utf8_lossy(¤t_head.stdout); - let target = String::from_utf8_lossy(&output.stdout); - current.trim() != target.trim() - } - Err(_) => { - // tag not found locally - return Err(format!( - "Tag {} not found in libpg_query submodule. Please run:\n\ - cd {} && git fetch --tags && git checkout {} && git checkout {} && git checkout {}", - libpg_query_tag, - libpg_query_submodule.display(), - "15-4.2.4", - "16-5.2.0", - "17-6.1.0" - ) - .into()); - } - }; - - if needs_checkout { - // checkout the correct tag for the selected postgresql version - println!( - "cargo:warning=Checking out libpg_query tag: {}", - libpg_query_tag - ); - let status = Command::new("git") - .args(["checkout", libpg_query_tag]) - .current_dir(&libpg_query_submodule) - .status()?; - - if !status.success() { - return Err(format!("Failed to checkout libpg_query tag: {}", libpg_query_tag).into()); - } - } - - // tell cargo to rerun if the submodule changes + // Tell cargo to rerun if the submodule changes println!( "cargo:rerun-if-changed={}", libpg_query_submodule.join("src").display() @@ -95,6 +38,7 @@ fn main() -> Result<(), Box> { let source_paths = vec![ libpg_query_submodule.join(LIBRARY_NAME).with_extension("h"), + libpg_query_submodule.join("postgres_deparse.h"), libpg_query_submodule.join("Makefile"), libpg_query_submodule.join("src"), libpg_query_submodule.join("protobuf"), diff --git a/crates/pgt_query/vendor/libpg_query b/crates/pgt_query/vendor/libpg_query index 1c1a32ed2..9ac12d29f 160000 --- a/crates/pgt_query/vendor/libpg_query +++ b/crates/pgt_query/vendor/libpg_query @@ -1 +1 @@ -Subproject commit 1c1a32ed2f4c7799830d50bf4cb159222aafec48 +Subproject commit 9ac12d29f2ffc0e49de8f4239223d50aaccd1549 diff --git a/crates/pgt_query_macros/build.rs b/crates/pgt_query_macros/build.rs index db83ce86e..235227ee3 100644 --- a/crates/pgt_query_macros/build.rs +++ b/crates/pgt_query_macros/build.rs @@ -3,47 +3,34 @@ use std::fs; use std::io::Write; use std::path::PathBuf; -// This should match the version used by pgt_query crate -// You can configure this via environment variable PG_QUERY_VERSION if needed -static LIBPG_QUERY_TAG: &str = "17-6.1.0"; +static LIBPG_QUERY_TAG: &str = "17-latest"; fn main() -> Result<(), Box> { - // Allow version override via environment variable - let version = env::var("PG_QUERY_VERSION").unwrap_or_else(|_| LIBPG_QUERY_TAG.to_string()); - - // Get the manifest directory (source directory) let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); - let postgres_dir = manifest_dir.join("postgres"); - let proto_filename = format!("{}.proto", version); - let proto_path = postgres_dir.join(&proto_filename); + let vendor_dir = manifest_dir.join("vendor").join(LIBPG_QUERY_TAG); + let proto_path = vendor_dir.join("pg_query.proto"); // Download proto file if not already present in source directory if !proto_path.exists() { println!( "cargo:warning=Downloading pg_query.proto for libpg_query {} to source directory", - version + LIBPG_QUERY_TAG ); - // Create postgres directory if it doesn't exist - fs::create_dir_all(&postgres_dir)?; + fs::create_dir_all(&vendor_dir)?; - // Download the proto file let proto_url = format!( "https://raw.githubusercontent.com/pganalyze/libpg_query/{}/protobuf/pg_query.proto", - version + LIBPG_QUERY_TAG ); let response = ureq::get(&proto_url).call()?; let proto_content = response.into_string()?; - // Write proto file to source directory let mut file = fs::File::create(&proto_path)?; file.write_all(proto_content.as_bytes())?; - println!( - "cargo:warning=Successfully downloaded pg_query.proto to {}", - proto_path.display() - ); + println!("cargo:warning=Successfully downloaded pg_query.proto to source"); } // Set environment variable for the proc macro diff --git a/crates/pgt_query_macros/postgres/17-6.1.0.proto b/crates/pgt_query_macros/vendor/17-latest/pg_query.proto similarity index 100% rename from crates/pgt_query_macros/postgres/17-6.1.0.proto rename to crates/pgt_query_macros/vendor/17-latest/pg_query.proto From 093b8e296baa621a437c388e6c12f20485bfb489 Mon Sep 17 00:00:00 2001 From: psteinroe Date: Sun, 31 Aug 2025 20:48:58 +0200 Subject: [PATCH 4/4] progress --- .gitmodules | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index e104f8b02..bd8a6acc3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,6 +5,4 @@ [submodule "crates/pgt_query/vendor/libpg_query"] path = crates/pgt_query/vendor/libpg_query url = https://github.com/pganalyze/libpg_query.git - fetchRecurseSubmodules = true - shallow = false branch = 17-latest