* feat: accept pdf, images and office files * refactor: rename FileConverter to FileParser, simplify file retain API - Rename engine/converters/ → engine/parsers/, FileConverter → FileParser, ConverterRegistry → FileParserRegistry, MarkitdownConverter → MarkitdownParser - Rename env var HINDSIGHT_API_FILE_CONVERTER → HINDSIGHT_API_FILE_PARSER - Remove async/document_tags params from FileRetainRequest (always async now) - Add retain_files() to Python Hindsight client and retainFiles() to TypeScript client - Add sample.pdf to doc examples for working file upload demonstrations - Update test_file_retain.py to use new parser names and always-async behavior - Fix Go client missing os import in api_files.go - Simplify postgresql.py storage to minimal schema * fix: update rust CLI tests to use is_supported_file instead of is_text_file * fix: patch Go api_files.go to add missing 'os' import after generation * fix: insert 'os' import after 'net/url' in api_files.go patch for correct position * chore: regenerate OpenAPI spec and clients (converter→parser description update)
182 lines
7.1 KiB
Rust
182 lines
7.1 KiB
Rust
use std::env;
|
|
use std::fs;
|
|
use std::path::PathBuf;
|
|
|
|
/// Convert OpenAPI 3.1 spec to 3.0 for progenitor compatibility
|
|
fn convert_31_to_30(spec: &mut serde_json::Value) {
|
|
// Change version from 3.1.x to 3.0.3
|
|
if let Some(obj) = spec.as_object_mut() {
|
|
obj.insert("openapi".to_string(), serde_json::json!("3.0.3"));
|
|
}
|
|
|
|
// Recursively convert anyOf with null to nullable
|
|
convert_anyof_to_nullable(spec);
|
|
}
|
|
|
|
/// Remove paths with multipart/form-data content type (not supported by progenitor)
|
|
fn filter_multipart_endpoints(spec: &mut serde_json::Value) {
|
|
if let Some(paths) = spec.get_mut("paths").and_then(|v| v.as_object_mut()) {
|
|
let mut paths_to_remove = Vec::new();
|
|
|
|
for (path_name, path_item) in paths.iter() {
|
|
if let Some(operations) = path_item.as_object() {
|
|
for (_method, operation) in operations.iter() {
|
|
if let Some(request_body) = operation.get("requestBody") {
|
|
if let Some(content) = request_body.get("content") {
|
|
if let Some(content_obj) = content.as_object() {
|
|
if content_obj.contains_key("multipart/form-data") {
|
|
eprintln!("Filtering out endpoint with multipart/form-data: {}", path_name);
|
|
paths_to_remove.push(path_name.clone());
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Remove the paths
|
|
for path in paths_to_remove {
|
|
paths.remove(&path);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn convert_anyof_to_nullable(value: &mut serde_json::Value) {
|
|
match value {
|
|
serde_json::Value::Object(obj) => {
|
|
// Check if this object has anyOf with null and process it
|
|
let should_convert = obj.get("anyOf")
|
|
.and_then(|v| v.as_array())
|
|
.map(|array| {
|
|
if array.len() == 2 {
|
|
let has_null = array.iter().any(|v| {
|
|
v.get("type")
|
|
.and_then(|t| t.as_str())
|
|
.map(|s| s == "null")
|
|
.unwrap_or(false)
|
|
});
|
|
has_null
|
|
} else {
|
|
false
|
|
}
|
|
})
|
|
.unwrap_or(false);
|
|
|
|
if should_convert {
|
|
// Clone the anyOf array to avoid borrow issues
|
|
if let Some(any_of) = obj.get("anyOf").cloned() {
|
|
if let Some(array) = any_of.as_array() {
|
|
// Find the non-null schema
|
|
if let Some(non_null_schema) = array.iter().find(|v| {
|
|
v.get("type")
|
|
.and_then(|t| t.as_str())
|
|
.map(|s| s != "null")
|
|
.unwrap_or(true)
|
|
}).cloned() {
|
|
// Replace anyOf with the non-null schema + nullable: true
|
|
obj.remove("anyOf");
|
|
if let Some(non_null_obj) = non_null_schema.as_object() {
|
|
for (k, v) in non_null_obj.iter() {
|
|
obj.insert(k.clone(), v.clone());
|
|
}
|
|
}
|
|
obj.insert("nullable".to_string(), serde_json::json!(true));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Recursively process all values
|
|
for (_key, val) in obj.iter_mut() {
|
|
convert_anyof_to_nullable(val);
|
|
}
|
|
}
|
|
serde_json::Value::Array(arr) => {
|
|
for item in arr.iter_mut() {
|
|
convert_anyof_to_nullable(item);
|
|
}
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
|
|
fn main() {
|
|
// Get the OpenAPI spec path from hindsight-docs/static (single source of truth)
|
|
let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
|
|
let openapi_path = manifest_dir
|
|
.parent()
|
|
.unwrap()
|
|
.parent()
|
|
.unwrap()
|
|
.join("hindsight-docs")
|
|
.join("static")
|
|
.join("openapi.json");
|
|
|
|
// Tell Cargo to rebuild if the OpenAPI spec changes
|
|
println!("cargo:rerun-if-changed={}", openapi_path.display());
|
|
|
|
// Read the OpenAPI spec
|
|
let spec_content = fs::read_to_string(&openapi_path)
|
|
.expect("Failed to read openapi.json. Make sure it exists in the project root.");
|
|
|
|
// Parse as generic JSON first to convert 3.1 to 3.0
|
|
let mut spec_json: serde_json::Value = serde_json::from_str(&spec_content)
|
|
.expect("Failed to parse openapi.json");
|
|
|
|
// Convert OpenAPI 3.1.0 to 3.0.3 for progenitor compatibility
|
|
if let Some(version) = spec_json.get("openapi").and_then(|v| v.as_str()) {
|
|
if version.starts_with("3.1") {
|
|
eprintln!("Converting OpenAPI 3.1 to 3.0 for compatibility...");
|
|
convert_31_to_30(&mut spec_json);
|
|
}
|
|
}
|
|
|
|
// Filter out multipart/form-data endpoints (progenitor doesn't support them)
|
|
filter_multipart_endpoints(&mut spec_json);
|
|
|
|
// Now parse as OpenAPI struct
|
|
let spec: openapiv3::OpenAPI = serde_json::from_value(spec_json)
|
|
.expect("Failed to parse converted OpenAPI spec");
|
|
|
|
// Generate the client
|
|
let mut generator = progenitor::Generator::default();
|
|
|
|
// Generate code
|
|
let tokens = generator.generate_tokens(&spec)
|
|
.expect("Failed to generate client code from OpenAPI spec");
|
|
|
|
// Write to the output directory
|
|
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
|
|
let dest_path = out_dir.join("hindsight_client_generated.rs");
|
|
|
|
let syntax_tree = syn::parse2(tokens)
|
|
.expect("Failed to parse generated tokens");
|
|
let mut formatted = prettyplease::unparse(&syntax_tree);
|
|
|
|
// Fix progenitor bug with optional header parameters
|
|
// The generated code tries to call .to_string() on Option<&str> which doesn't work
|
|
// We need to unwrap the Option first
|
|
formatted = fix_optional_header_params(&formatted);
|
|
|
|
fs::write(&dest_path, formatted)
|
|
.expect("Failed to write generated client code");
|
|
|
|
println!("Generated client at: {}", dest_path.display());
|
|
}
|
|
|
|
/// Fix progenitor's generated code for optional header parameters
|
|
/// Replaces patterns like `value.to_string().try_into()?` where value is Option<&str>
|
|
/// with `value.unwrap_or_default().to_string().try_into()?`
|
|
fn fix_optional_header_params(code: &str) -> String {
|
|
use regex::Regex;
|
|
|
|
// Pattern: header_map.append("authorization", value.to_string().try_into()?);
|
|
// Should become: header_map.append("authorization", value.unwrap_or_default().to_string().try_into()?);
|
|
let re = Regex::new(r#"header_map\.append\("authorization", value\.to_string\(\)\.try_into\(\)\?\)"#)
|
|
.expect("Invalid regex");
|
|
|
|
re.replace_all(code, r#"header_map.append("authorization", value.unwrap_or_default().to_string().try_into()?)"#)
|
|
.to_string()
|
|
}
|