chore: offline builds with submodules

.gitmodules

@@ -2,3 +2,7 @@
 	path = lib/tree_sitter_sql/tree-sitter-sql
 	url = https://github.com/DerekStride/tree-sitter-sql
 	branch = gh-pages
+[submodule "crates/pgt_query/vendor/libpg_query"]
+	path = crates/pgt_query/vendor/libpg_query
+	url = https://github.com/pganalyze/libpg_query.git
+	branch = 17-latest

crates/pgt_lexer_codegen/build.rs

@@ -3,39 +3,34 @@ use std::fs;
 use std::io::Write;
 use std::path::PathBuf;
 
-// TODO make this selectable via feature flags
-static LIBPG_QUERY_TAG: &str = "17-6.1.0";
+static LIBPG_QUERY_TAG: &str = "17-latest";
 
-/// Downloads the `kwlist.h` file from the specified version of `libpg_query`
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let version = LIBPG_QUERY_TAG.to_string();
-
-    // Check for the postgres header file in the source tree first
     let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?);
-    let headers_dir = manifest_dir.join("postgres").join(&version);
-    let kwlist_path = headers_dir.join("kwlist.h");
+    let vendor_dir = manifest_dir.join("vendor").join(LIBPG_QUERY_TAG);
+    let kwlist_path = vendor_dir.join("kwlist.h");
 
-    // Only download if the file doesn't exist
+    // Download kwlist.h if not already present in source directory
     if !kwlist_path.exists() {
         println!(
-            "cargo:warning=Downloading kwlist.h for libpg_query {}",
-            version
+            "cargo:warning=Downloading kwlist.h for libpg_query {} to source directory",
+            LIBPG_QUERY_TAG
         );
 
-        fs::create_dir_all(&headers_dir)?;
+        fs::create_dir_all(&vendor_dir)?;
 
-        let proto_url = format!(
+        let kwlist_url = format!(
             "https://raw.githubusercontent.com/pganalyze/libpg_query/{}/src/postgres/include/parser/kwlist.h",
-            version
+            LIBPG_QUERY_TAG
         );
 
-        let response = ureq::get(&proto_url).call()?;
+        let response = ureq::get(&kwlist_url).call()?;
         let content = response.into_string()?;
 
         let mut file = fs::File::create(&kwlist_path)?;
         file.write_all(content.as_bytes())?;
 
-        println!("cargo:warning=Successfully downloaded kwlist.h");
+        println!("cargo:warning=Successfully downloaded kwlist.h to source");
     }
 
     println!(

crates/pgt_query/Cargo.toml

@@ -18,10 +18,7 @@ pgt_query_macros = { workspace = true }
 
 
 [features]
-default     = ["postgres-17"]
-postgres-15 = []
-postgres-16 = []
-postgres-17 = []
+default = []
 
 [build-dependencies]
 bindgen     = "0.72.0"

crates/pgt_query/build.rs

@@ -1,80 +1,48 @@
-#![cfg_attr(feature = "clippy", feature(plugin))]
-#![cfg_attr(feature = "clippy", plugin(clippy))]
-
 use fs_extra::dir::CopyOptions;
 use glob::glob;
 use std::env;
 use std::path::PathBuf;
 use std::process::Command;
 
 static LIBRARY_NAME: &str = "pg_query";
-static LIBPG_QUERY_REPO: &str = "https://github.com/pganalyze/libpg_query.git";
-fn get_libpg_query_tag() -> &'static str {
-    #[cfg(feature = "postgres-15")]
-    return "15-5.3.0";
-    #[cfg(feature = "postgres-16")]
-    return "16-6.1.0";
-    #[cfg(feature = "postgres-17")]
-    return "17-6.1.0";
-}
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let libpg_query_tag = get_libpg_query_tag();
     let out_dir = PathBuf::from(env::var("OUT_DIR")?);
-    let vendor_dir = out_dir.join("vendor");
-    let libpg_query_dir = vendor_dir.join("libpg_query").join(libpg_query_tag);
-    let stamp_file = libpg_query_dir.join(".stamp");
+    let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?);
+    let libpg_query_submodule = manifest_dir.join("vendor").join("libpg_query");
 
-    let src_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).join("src");
+    let src_dir = manifest_dir.join("src");
     let target = env::var("TARGET").unwrap();
     let is_emscripten = target.contains("emscripten");
 
-    // Configure cargo through stdout
     println!("cargo:rustc-link-search=native={}", out_dir.display());
     println!("cargo:rustc-link-lib=static={LIBRARY_NAME}");
 
-    // Clone libpg_query if not already present
-    if !stamp_file.exists() {
-        println!("cargo:warning=Cloning libpg_query {}", libpg_query_tag);
-
-        // Create vendor directory
-        std::fs::create_dir_all(&vendor_dir)?;
-
-        // Clone the repository with partial clone for faster download
-        let status = Command::new("git")
-            .args([
-                "clone",
-                "--filter=blob:none",
-                "--depth",
-                "1",
-                "--branch",
-                libpg_query_tag,
-                LIBPG_QUERY_REPO,
-                libpg_query_dir.to_str().unwrap(),
-            ])
-            .status()?;
-
-        if !status.success() {
-            return Err("Failed to clone libpg_query".into());
-        }
-
-        // Create stamp file
-        std::fs::File::create(&stamp_file)?;
+    // Check if submodule exists
+    if !libpg_query_submodule.join(".git").exists() && !libpg_query_submodule.join("src").exists() {
+        return Err(
+            "libpg_query submodule not found. Please run: git submodule update --init --recursive"
+                .into(),
+        );
     }
 
-    // Tell cargo to rerun if the stamp file is deleted
-    println!("cargo:rerun-if-changed={}", stamp_file.display());
+    // Tell cargo to rerun if the submodule changes
+    println!(
+        "cargo:rerun-if-changed={}",
+        libpg_query_submodule.join("src").display()
+    );
 
-    // Copy necessary files to OUT_DIR for compilation
+    // copy necessary files to out_dir for compilation
     let out_header_path = out_dir.join(LIBRARY_NAME).with_extension("h");
     let out_protobuf_path = out_dir.join("protobuf");
 
     let source_paths = vec![
-        libpg_query_dir.join(LIBRARY_NAME).with_extension("h"),
-        libpg_query_dir.join("Makefile"),
-        libpg_query_dir.join("src"),
-        libpg_query_dir.join("protobuf"),
-        libpg_query_dir.join("vendor"),
+        libpg_query_submodule.join(LIBRARY_NAME).with_extension("h"),
+        libpg_query_submodule.join("postgres_deparse.h"),
+        libpg_query_submodule.join("Makefile"),
+        libpg_query_submodule.join("src"),
+        libpg_query_submodule.join("protobuf"),
+        libpg_query_submodule.join("vendor"),
     ];
 
     let copy_options = CopyOptions {
@@ -84,17 +52,17 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     fs_extra::copy_items(&source_paths, &out_dir, &copy_options)?;
 
-    // Compile the C library.
+    // compile the c library.
     let mut build = cc::Build::new();
 
-    // Configure for Emscripten if needed
+    // configure for emscripten if needed
     if is_emscripten {
-        // Use emcc as the compiler instead of gcc/clang
+        // use emcc as the compiler instead of gcc/clang
         build.compiler("emcc");
-        // Use emar as the archiver instead of ar
+        // use emar as the archiver instead of ar
         build.archiver("emar");
-        // Note: We don't add WASM-specific flags here as this creates a static library
-        // The final linking flags should be added when building the final WASM module
+        // note: we don't add wasm-specific flags here as this creates a static library
+        // the final linking flags should be added when building the final wasm module
     }
 
     build
@@ -115,7 +83,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .include(out_dir.join("./vendor"))
         .include(out_dir.join("./src/postgres/include"))
         .include(out_dir.join("./src/include"))
-        .warnings(false); // Avoid unnecessary warnings, as they are already considered as part of libpg_query development
+        .warnings(false); // avoid unnecessary warnings, as they are already considered as part of libpg_query development
     if env::var("PROFILE").unwrap() == "debug" || env::var("DEBUG").unwrap() == "1" {
         build.define("USE_ASSERT_CHECKING", None);
     }

crates/pgt_query_macros/build.rs

@@ -3,47 +3,34 @@ use std::fs;
 use std::io::Write;
 use std::path::PathBuf;
 
-// This should match the version used by pgt_query crate
-// You can configure this via environment variable PG_QUERY_VERSION if needed
-static LIBPG_QUERY_TAG: &str = "17-6.1.0";
+static LIBPG_QUERY_TAG: &str = "17-latest";
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Allow version override via environment variable
-    let version = env::var("PG_QUERY_VERSION").unwrap_or_else(|_| LIBPG_QUERY_TAG.to_string());
-
-    // Get the manifest directory (source directory)
     let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?);
-    let postgres_dir = manifest_dir.join("postgres");
-    let proto_filename = format!("{}.proto", version);
-    let proto_path = postgres_dir.join(&proto_filename);
+    let vendor_dir = manifest_dir.join("vendor").join(LIBPG_QUERY_TAG);
+    let proto_path = vendor_dir.join("pg_query.proto");
 
     // Download proto file if not already present in source directory
     if !proto_path.exists() {
         println!(
             "cargo:warning=Downloading pg_query.proto for libpg_query {} to source directory",
-            version
+            LIBPG_QUERY_TAG
         );
 
-        // Create postgres directory if it doesn't exist
-        fs::create_dir_all(&postgres_dir)?;
+        fs::create_dir_all(&vendor_dir)?;
 
-        // Download the proto file
         let proto_url = format!(
             "https://raw.githubusercontent.com/pganalyze/libpg_query/{}/protobuf/pg_query.proto",
-            version
+            LIBPG_QUERY_TAG
         );
 
         let response = ureq::get(&proto_url).call()?;
         let proto_content = response.into_string()?;
 
-        // Write proto file to source directory
         let mut file = fs::File::create(&proto_path)?;
         file.write_all(proto_content.as_bytes())?;
 
-        println!(
-            "cargo:warning=Successfully downloaded pg_query.proto to {}",
-            proto_path.display()
-        );
+        println!("cargo:warning=Successfully downloaded pg_query.proto to source");
     }
 
     // Set environment variable for the proc macro