* feat: allow per-request file parser selection with fallback chains Clients can now specify which parser(s) to use when calling the file retain endpoint, instead of being locked to the server-side default. Changes: - `parser` field added to `FileRetainRequest` (request-level default) and `FileRetainMetadata` (per-file override); accepts a single name or an ordered fallback chain (list) - Resolution priority: per-file > request-level > server default - `HINDSIGHT_API_FILE_PARSER` now accepts a comma-separated fallback chain (e.g. `iris,markitdown`); fully backward-compatible - New `HINDSIGHT_API_FILE_PARSER_ALLOWLIST` env var restricts which parsers clients may request (defaults to all registered parsers) - Invalid/disallowed parser names are rejected with HTTP 400 - `FileParserRegistry.convert_with_fallback()` tries each parser in order, falling back on UnsupportedFileTypeError, empty content, or any other error - Worker updated to use the fallback chain stored per-task - OpenAPI spec and all generated clients regenerated * fix: handle on_file_convert_complete hook and rebase onto main - Return ConvertResult dataclass from convert_with_fallback() instead of a plain str, carrying both the content and the winning parser name - Use winning_parser_name in the on_file_convert_complete hook so parser_name reflects the parser that actually succeeded, not the chain - Update all test calls to submit_async_file_retain() to use the new per-item parser field instead of the removed top-level parser= kwarg * docs: document HINDSIGHT_API_FILE_PARSER fallback chain and ALLOWLIST
214 lines
6.9 KiB
Go
214 lines
6.9 KiB
Go
/*
|
|
Hindsight HTTP API
|
|
|
|
HTTP API for Hindsight
|
|
|
|
API version: 0.4.16
|
|
*/
|
|
|
|
// Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
|
|
|
|
package hindsight
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"strings"
|
|
)
|
|
|
|
|
|
// FilesAPIService FilesAPI service
|
|
type FilesAPIService service
|
|
|
|
type ApiFileRetainRequest struct {
|
|
ctx context.Context
|
|
ApiService *FilesAPIService
|
|
bankId string
|
|
files []*os.File
|
|
request *string
|
|
authorization *string
|
|
}
|
|
|
|
// Files to upload and convert
|
|
func (r ApiFileRetainRequest) Files(files []*os.File) ApiFileRetainRequest {
|
|
r.files = files
|
|
return r
|
|
}
|
|
|
|
// JSON string with FileRetainRequest model
|
|
func (r ApiFileRetainRequest) Request(request string) ApiFileRetainRequest {
|
|
r.request = &request
|
|
return r
|
|
}
|
|
|
|
func (r ApiFileRetainRequest) Authorization(authorization string) ApiFileRetainRequest {
|
|
r.authorization = &authorization
|
|
return r
|
|
}
|
|
|
|
func (r ApiFileRetainRequest) Execute() (*FileRetainResponse, *http.Response, error) {
|
|
return r.ApiService.FileRetainExecute(r)
|
|
}
|
|
|
|
/*
|
|
FileRetain Convert files to memories
|
|
|
|
Upload files (PDF, DOCX, etc.), convert them to markdown, and retain as memories.
|
|
|
|
This endpoint handles file upload, conversion, and memory creation in a single operation.
|
|
|
|
**Features:**
|
|
- Supports PDF, DOCX, PPTX, XLSX, images (with OCR), audio (with transcription)
|
|
- Automatic file-to-markdown conversion using pluggable parsers
|
|
- Files stored in object storage (PostgreSQL by default, S3 for production)
|
|
- Each file becomes a separate document with optional metadata/tags
|
|
- Always processes asynchronously — returns operation IDs immediately
|
|
|
|
**The system automatically:**
|
|
1. Stores uploaded files in object storage
|
|
2. Converts files to markdown
|
|
3. Creates document records with file metadata
|
|
4. Extracts facts and creates memory units (same as regular retain)
|
|
|
|
Use the operations endpoint to monitor progress.
|
|
|
|
**Request format:** multipart/form-data with:
|
|
- `files`: One or more files to upload
|
|
- `request`: JSON string with FileRetainRequest model
|
|
|
|
**Parser selection:**
|
|
- Set `parser` in the request body to override the server default for all files.
|
|
- Set `parser` inside a `files_metadata` entry for per-file control.
|
|
- Pass a list (e.g. `['iris', 'markitdown']`) to define an ordered fallback chain — each parser is tried in sequence until one succeeds.
|
|
- Falls back to the server default (`HINDSIGHT_API_FILE_PARSER`) if not specified.
|
|
- Only parsers enabled on the server may be requested; others return HTTP 400.
|
|
|
|
@param ctx context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
|
|
@param bankId
|
|
@return ApiFileRetainRequest
|
|
*/
|
|
func (a *FilesAPIService) FileRetain(ctx context.Context, bankId string) ApiFileRetainRequest {
|
|
return ApiFileRetainRequest{
|
|
ApiService: a,
|
|
ctx: ctx,
|
|
bankId: bankId,
|
|
}
|
|
}
|
|
|
|
// Execute executes the request
|
|
// @return FileRetainResponse
|
|
func (a *FilesAPIService) FileRetainExecute(r ApiFileRetainRequest) (*FileRetainResponse, *http.Response, error) {
|
|
var (
|
|
localVarHTTPMethod = http.MethodPost
|
|
localVarPostBody interface{}
|
|
formFiles []formFile
|
|
localVarReturnValue *FileRetainResponse
|
|
)
|
|
|
|
localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "FilesAPIService.FileRetain")
|
|
if err != nil {
|
|
return localVarReturnValue, nil, &GenericOpenAPIError{error: err.Error()}
|
|
}
|
|
|
|
localVarPath := localBasePath + "/v1/default/banks/{bank_id}/files/retain"
|
|
localVarPath = strings.Replace(localVarPath, "{"+"bank_id"+"}", url.PathEscape(parameterValueToString(r.bankId, "bankId")), -1)
|
|
|
|
localVarHeaderParams := make(map[string]string)
|
|
localVarQueryParams := url.Values{}
|
|
localVarFormParams := url.Values{}
|
|
if r.files == nil {
|
|
return localVarReturnValue, nil, reportError("files is required and must be specified")
|
|
}
|
|
if r.request == nil {
|
|
return localVarReturnValue, nil, reportError("request is required and must be specified")
|
|
}
|
|
|
|
// to determine the Content-Type header
|
|
localVarHTTPContentTypes := []string{"multipart/form-data"}
|
|
|
|
// set Content-Type header
|
|
localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
|
|
if localVarHTTPContentType != "" {
|
|
localVarHeaderParams["Content-Type"] = localVarHTTPContentType
|
|
}
|
|
|
|
// to determine the Accept header
|
|
localVarHTTPHeaderAccepts := []string{"application/json"}
|
|
|
|
// set Accept header
|
|
localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
|
|
if localVarHTTPHeaderAccept != "" {
|
|
localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
|
|
}
|
|
if r.authorization != nil {
|
|
parameterAddToHeaderOrQuery(localVarHeaderParams, "authorization", r.authorization, "simple", "")
|
|
}
|
|
var filesLocalVarFormFileName string
|
|
var filesLocalVarFileName string
|
|
var filesLocalVarFileBytes []byte
|
|
|
|
filesLocalVarFormFileName = "files"
|
|
filesLocalVarFile := r.files
|
|
|
|
if filesLocalVarFile != nil {
|
|
// loop through the array to prepare multiple files upload
|
|
for _, filesLocalVarFileValue := range filesLocalVarFile {
|
|
fbs, _ := io.ReadAll(filesLocalVarFileValue)
|
|
|
|
filesLocalVarFileBytes = fbs
|
|
filesLocalVarFileName = filesLocalVarFileValue.Name()
|
|
filesLocalVarFileValue.Close()
|
|
formFiles = append(formFiles, formFile{fileBytes: filesLocalVarFileBytes, fileName: filesLocalVarFileName, formFileName: filesLocalVarFormFileName})
|
|
}
|
|
}
|
|
parameterAddToHeaderOrQuery(localVarFormParams, "request", r.request, "", "")
|
|
req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, formFiles)
|
|
if err != nil {
|
|
return localVarReturnValue, nil, err
|
|
}
|
|
|
|
localVarHTTPResponse, err := a.client.callAPI(req)
|
|
if err != nil || localVarHTTPResponse == nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
localVarBody, err := io.ReadAll(localVarHTTPResponse.Body)
|
|
localVarHTTPResponse.Body.Close()
|
|
localVarHTTPResponse.Body = io.NopCloser(bytes.NewBuffer(localVarBody))
|
|
if err != nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
if localVarHTTPResponse.StatusCode >= 300 {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: localVarHTTPResponse.Status,
|
|
}
|
|
if localVarHTTPResponse.StatusCode == 422 {
|
|
var v HTTPValidationError
|
|
err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr.error = err.Error()
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
newErr.error = formatErrorMessage(localVarHTTPResponse.Status, &v)
|
|
newErr.model = v
|
|
}
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: err.Error(),
|
|
}
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
return localVarReturnValue, localVarHTTPResponse, nil
|
|
}
|