diff --git a/.gitmodules b/.gitmodules index 9b62ce889..bd8a6acc3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,7 @@ path = lib/tree_sitter_sql/tree-sitter-sql url = https://github.com/DerekStride/tree-sitter-sql branch = gh-pages +[submodule "crates/pgt_query/vendor/libpg_query"] + path = crates/pgt_query/vendor/libpg_query + url = https://github.com/pganalyze/libpg_query.git + branch = 17-latest diff --git a/Cargo.lock b/Cargo.lock index be7eacb75..7011a88da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3863,9 +3863,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.28" +version = "0.23.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" +checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" dependencies = [ "log", "once_cell", @@ -3887,9 +3887,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.3" +version = "0.103.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" +checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc" dependencies = [ "ring", "rustls-pki-types", @@ -5325,14 +5325,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.1", + "webpki-roots 1.0.2", ] [[package]] name = "webpki-roots" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8782dd5a41a24eed3a4f40b606249b3e236ca61adf1f25ea4d45c73de122b502" +checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" dependencies = [ "rustls-pki-types", ] diff --git a/crates/pgt_lexer_codegen/build.rs b/crates/pgt_lexer_codegen/build.rs index 70c9635d4..fc15b7749 100644 --- a/crates/pgt_lexer_codegen/build.rs +++ b/crates/pgt_lexer_codegen/build.rs @@ -3,39 +3,34 @@ use std::fs; use std::io::Write; use std::path::PathBuf; -// TODO make this selectable via feature flags -static LIBPG_QUERY_TAG: &str = "17-6.1.0"; +static LIBPG_QUERY_TAG: &str = "17-latest"; -/// Downloads the `kwlist.h` file from the specified version of `libpg_query` fn main() -> Result<(), Box> { - let version = LIBPG_QUERY_TAG.to_string(); - - // Check for the postgres header file in the source tree first let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); - let headers_dir = manifest_dir.join("postgres").join(&version); - let kwlist_path = headers_dir.join("kwlist.h"); + let vendor_dir = manifest_dir.join("vendor").join(LIBPG_QUERY_TAG); + let kwlist_path = vendor_dir.join("kwlist.h"); - // Only download if the file doesn't exist + // Download kwlist.h if not already present in source directory if !kwlist_path.exists() { println!( - "cargo:warning=Downloading kwlist.h for libpg_query {}", - version + "cargo:warning=Downloading kwlist.h for libpg_query {} to source directory", + LIBPG_QUERY_TAG ); - fs::create_dir_all(&headers_dir)?; + fs::create_dir_all(&vendor_dir)?; - let proto_url = format!( + let kwlist_url = format!( "https://raw.githubusercontent.com/pganalyze/libpg_query/{}/src/postgres/include/parser/kwlist.h", - version + LIBPG_QUERY_TAG ); - let response = ureq::get(&proto_url).call()?; + let response = ureq::get(&kwlist_url).call()?; let content = response.into_string()?; let mut file = fs::File::create(&kwlist_path)?; file.write_all(content.as_bytes())?; - println!("cargo:warning=Successfully downloaded kwlist.h"); + println!("cargo:warning=Successfully downloaded kwlist.h to source"); } println!( diff --git a/crates/pgt_lexer_codegen/postgres/17-6.1.0/kwlist.h b/crates/pgt_lexer_codegen/vendor/17-latest/kwlist.h similarity index 100% rename from crates/pgt_lexer_codegen/postgres/17-6.1.0/kwlist.h rename to crates/pgt_lexer_codegen/vendor/17-latest/kwlist.h diff --git a/crates/pgt_query/Cargo.toml b/crates/pgt_query/Cargo.toml index 881b1b800..8fdbe6ca1 100644 --- a/crates/pgt_query/Cargo.toml +++ b/crates/pgt_query/Cargo.toml @@ -18,10 +18,7 @@ pgt_query_macros = { workspace = true } [features] -default = ["postgres-17"] -postgres-15 = [] -postgres-16 = [] -postgres-17 = [] +default = [] [build-dependencies] bindgen = "0.72.0" diff --git a/crates/pgt_query/build.rs b/crates/pgt_query/build.rs index 292b3af28..6570746bc 100644 --- a/crates/pgt_query/build.rs +++ b/crates/pgt_query/build.rs @@ -1,6 +1,3 @@ -#![cfg_attr(feature = "clippy", feature(plugin))] -#![cfg_attr(feature = "clippy", plugin(clippy))] - use fs_extra::dir::CopyOptions; use glob::glob; use std::env; @@ -8,73 +5,44 @@ use std::path::PathBuf; use std::process::Command; static LIBRARY_NAME: &str = "pg_query"; -static LIBPG_QUERY_REPO: &str = "https://github.com/pganalyze/libpg_query.git"; -fn get_libpg_query_tag() -> &'static str { - #[cfg(feature = "postgres-15")] - return "15-5.3.0"; - #[cfg(feature = "postgres-16")] - return "16-6.1.0"; - #[cfg(feature = "postgres-17")] - return "17-6.1.0"; -} fn main() -> Result<(), Box> { - let libpg_query_tag = get_libpg_query_tag(); let out_dir = PathBuf::from(env::var("OUT_DIR")?); - let vendor_dir = out_dir.join("vendor"); - let libpg_query_dir = vendor_dir.join("libpg_query").join(libpg_query_tag); - let stamp_file = libpg_query_dir.join(".stamp"); + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); + let libpg_query_submodule = manifest_dir.join("vendor").join("libpg_query"); - let src_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).join("src"); + let src_dir = manifest_dir.join("src"); let target = env::var("TARGET").unwrap(); let is_emscripten = target.contains("emscripten"); - // Configure cargo through stdout println!("cargo:rustc-link-search=native={}", out_dir.display()); println!("cargo:rustc-link-lib=static={LIBRARY_NAME}"); - // Clone libpg_query if not already present - if !stamp_file.exists() { - println!("cargo:warning=Cloning libpg_query {}", libpg_query_tag); - - // Create vendor directory - std::fs::create_dir_all(&vendor_dir)?; - - // Clone the repository with partial clone for faster download - let status = Command::new("git") - .args([ - "clone", - "--filter=blob:none", - "--depth", - "1", - "--branch", - libpg_query_tag, - LIBPG_QUERY_REPO, - libpg_query_dir.to_str().unwrap(), - ]) - .status()?; - - if !status.success() { - return Err("Failed to clone libpg_query".into()); - } - - // Create stamp file - std::fs::File::create(&stamp_file)?; + // Check if submodule exists + if !libpg_query_submodule.join(".git").exists() && !libpg_query_submodule.join("src").exists() { + return Err( + "libpg_query submodule not found. Please run: git submodule update --init --recursive" + .into(), + ); } - // Tell cargo to rerun if the stamp file is deleted - println!("cargo:rerun-if-changed={}", stamp_file.display()); + // Tell cargo to rerun if the submodule changes + println!( + "cargo:rerun-if-changed={}", + libpg_query_submodule.join("src").display() + ); - // Copy necessary files to OUT_DIR for compilation + // copy necessary files to out_dir for compilation let out_header_path = out_dir.join(LIBRARY_NAME).with_extension("h"); let out_protobuf_path = out_dir.join("protobuf"); let source_paths = vec![ - libpg_query_dir.join(LIBRARY_NAME).with_extension("h"), - libpg_query_dir.join("Makefile"), - libpg_query_dir.join("src"), - libpg_query_dir.join("protobuf"), - libpg_query_dir.join("vendor"), + libpg_query_submodule.join(LIBRARY_NAME).with_extension("h"), + libpg_query_submodule.join("postgres_deparse.h"), + libpg_query_submodule.join("Makefile"), + libpg_query_submodule.join("src"), + libpg_query_submodule.join("protobuf"), + libpg_query_submodule.join("vendor"), ]; let copy_options = CopyOptions { @@ -84,17 +52,17 @@ fn main() -> Result<(), Box> { fs_extra::copy_items(&source_paths, &out_dir, ©_options)?; - // Compile the C library. + // compile the c library. let mut build = cc::Build::new(); - // Configure for Emscripten if needed + // configure for emscripten if needed if is_emscripten { - // Use emcc as the compiler instead of gcc/clang + // use emcc as the compiler instead of gcc/clang build.compiler("emcc"); - // Use emar as the archiver instead of ar + // use emar as the archiver instead of ar build.archiver("emar"); - // Note: We don't add WASM-specific flags here as this creates a static library - // The final linking flags should be added when building the final WASM module + // note: we don't add wasm-specific flags here as this creates a static library + // the final linking flags should be added when building the final wasm module } build @@ -115,7 +83,7 @@ fn main() -> Result<(), Box> { .include(out_dir.join("./vendor")) .include(out_dir.join("./src/postgres/include")) .include(out_dir.join("./src/include")) - .warnings(false); // Avoid unnecessary warnings, as they are already considered as part of libpg_query development + .warnings(false); // avoid unnecessary warnings, as they are already considered as part of libpg_query development if env::var("PROFILE").unwrap() == "debug" || env::var("DEBUG").unwrap() == "1" { build.define("USE_ASSERT_CHECKING", None); } diff --git a/crates/pgt_query/vendor/libpg_query b/crates/pgt_query/vendor/libpg_query new file mode 160000 index 000000000..9ac12d29f --- /dev/null +++ b/crates/pgt_query/vendor/libpg_query @@ -0,0 +1 @@ +Subproject commit 9ac12d29f2ffc0e49de8f4239223d50aaccd1549 diff --git a/crates/pgt_query_macros/build.rs b/crates/pgt_query_macros/build.rs index db83ce86e..235227ee3 100644 --- a/crates/pgt_query_macros/build.rs +++ b/crates/pgt_query_macros/build.rs @@ -3,47 +3,34 @@ use std::fs; use std::io::Write; use std::path::PathBuf; -// This should match the version used by pgt_query crate -// You can configure this via environment variable PG_QUERY_VERSION if needed -static LIBPG_QUERY_TAG: &str = "17-6.1.0"; +static LIBPG_QUERY_TAG: &str = "17-latest"; fn main() -> Result<(), Box> { - // Allow version override via environment variable - let version = env::var("PG_QUERY_VERSION").unwrap_or_else(|_| LIBPG_QUERY_TAG.to_string()); - - // Get the manifest directory (source directory) let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); - let postgres_dir = manifest_dir.join("postgres"); - let proto_filename = format!("{}.proto", version); - let proto_path = postgres_dir.join(&proto_filename); + let vendor_dir = manifest_dir.join("vendor").join(LIBPG_QUERY_TAG); + let proto_path = vendor_dir.join("pg_query.proto"); // Download proto file if not already present in source directory if !proto_path.exists() { println!( "cargo:warning=Downloading pg_query.proto for libpg_query {} to source directory", - version + LIBPG_QUERY_TAG ); - // Create postgres directory if it doesn't exist - fs::create_dir_all(&postgres_dir)?; + fs::create_dir_all(&vendor_dir)?; - // Download the proto file let proto_url = format!( "https://raw.githubusercontent.com/pganalyze/libpg_query/{}/protobuf/pg_query.proto", - version + LIBPG_QUERY_TAG ); let response = ureq::get(&proto_url).call()?; let proto_content = response.into_string()?; - // Write proto file to source directory let mut file = fs::File::create(&proto_path)?; file.write_all(proto_content.as_bytes())?; - println!( - "cargo:warning=Successfully downloaded pg_query.proto to {}", - proto_path.display() - ); + println!("cargo:warning=Successfully downloaded pg_query.proto to source"); } // Set environment variable for the proc macro diff --git a/crates/pgt_query_macros/postgres/17-6.1.0.proto b/crates/pgt_query_macros/vendor/17-latest/pg_query.proto similarity index 100% rename from crates/pgt_query_macros/postgres/17-6.1.0.proto rename to crates/pgt_query_macros/vendor/17-latest/pg_query.proto