From dcaacbe407aea3e3b23d3f852121a489fd8fb2d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Tue, 10 Mar 2026 17:16:11 +0100 Subject: [PATCH] feat: add manual retry for failed async operations (#537) - API: POST /v1/default/banks/{bank_id}/operations/{operation_id}/retry resets status to pending so the worker re-executes the task - UI: Retry button on failed operations in the operations view - Control plane proxy route + ControlPlaneClient.retryOperation() - Updated OpenAPI spec, all generated clients, and operations docs --- hindsight-api/hindsight_api/api/http.py | 51 +++ .../hindsight_api/engine/memory_engine.py | 58 ++++ hindsight-clients/go/api/openapi.yaml | 66 ++++ hindsight-clients/go/api_operations.go | 126 ++++++++ .../go/model_retry_operation_response.go | 214 +++++++++++++ .../python/.openapi-generator/FILES | 1 + .../python/hindsight_client_api/__init__.py | 1 + .../api/operations_api.py | 294 ++++++++++++++++++ .../hindsight_client_api/models/__init__.py | 1 + .../models/retry_operation_response.py | 91 ++++++ .../typescript/generated/sdk.gen.ts | 20 ++ .../typescript/generated/types.gen.ts | 62 ++++ .../operations/[operationId]/route.ts | 39 ++- .../src/components/bank-operations-view.tsx | 36 ++- hindsight-control-plane/src/lib/api.ts | 13 + .../docs/developer/api/operations.md | 27 ++ hindsight-docs/static/openapi.json | 97 ++++++ 17 files changed, 1195 insertions(+), 2 deletions(-) create mode 100644 hindsight-clients/go/model_retry_operation_response.go create mode 100644 hindsight-clients/python/hindsight_client_api/models/retry_operation_response.py diff --git a/hindsight-api/hindsight_api/api/http.py b/hindsight-api/hindsight_api/api/http.py index a4b9893e..aee66f82 100644 --- a/hindsight-api/hindsight_api/api/http.py +++ b/hindsight-api/hindsight_api/api/http.py @@ -1558,6 +1558,24 @@ class CancelOperationResponse(BaseModel): operation_id: str +class RetryOperationResponse(BaseModel): + """Response model for retry operation endpoint.""" + + model_config = ConfigDict( + json_schema_extra={ + "example": { + "success": True, + "message": "Operation 550e8400-e29b-41d4-a716-446655440000 queued for retry", + "operation_id": "550e8400-e29b-41d4-a716-446655440000", + } + } + ) + + success: bool + message: str + operation_id: str + + class ChildOperationStatus(BaseModel): """Status of a child operation (for batch operations).""" @@ -3532,6 +3550,39 @@ def _register_routes(app: FastAPI): logger.error(f"Error in /v1/default/banks/{bank_id}/operations/{operation_id}: {error_detail}") raise HTTPException(status_code=500, detail=str(e)) + @app.post( + "/v1/default/banks/{bank_id}/operations/{operation_id}/retry", + response_model=RetryOperationResponse, + summary="Retry a failed async operation", + description="Re-queue a failed async operation so the worker picks it up again", + operation_id="retry_operation", + tags=["Operations"], + ) + async def api_retry_operation( + bank_id: str, operation_id: str, request_context: RequestContext = Depends(get_request_context) + ): + """Retry a failed async operation.""" + try: + try: + uuid.UUID(operation_id) + except ValueError: + raise HTTPException(status_code=400, detail=f"Invalid operation_id format: {operation_id}") + + result = await app.state.memory.retry_operation(bank_id, operation_id, request_context=request_context) + return RetryOperationResponse(**result) + except ValueError as e: + raise HTTPException(status_code=404, detail=str(e)) + except OperationValidationError as e: + raise HTTPException(status_code=e.status_code, detail=e.reason) + except (AuthenticationError, HTTPException): + raise + except Exception as e: + import traceback + + error_detail = f"{str(e)}\n\nTraceback:\n{traceback.format_exc()}" + logger.error(f"Error in POST /v1/default/banks/{bank_id}/operations/{operation_id}/retry: {error_detail}") + raise HTTPException(status_code=500, detail=str(e)) + @app.get( "/v1/default/banks/{bank_id}/profile", response_model=BankProfileResponse, diff --git a/hindsight-api/hindsight_api/engine/memory_engine.py b/hindsight-api/hindsight_api/engine/memory_engine.py index 4bbec15b..a4baa56c 100644 --- a/hindsight-api/hindsight_api/engine/memory_engine.py +++ b/hindsight-api/hindsight_api/engine/memory_engine.py @@ -7118,6 +7118,64 @@ class MemoryEngine(MemoryEngineInterface): "bank_id": bank_id, } + async def retry_operation( + self, + bank_id: str, + operation_id: str, + *, + request_context: "RequestContext", + ) -> dict[str, Any]: + """Re-queue a failed async operation.""" + await self._authenticate_tenant(request_context) + from hindsight_api.extensions import OperationValidationError + + if self._operation_validator: + from hindsight_api.extensions import BankWriteContext + + ctx = BankWriteContext(bank_id=bank_id, operation="retry_operation", request_context=request_context) + await self._validate_operation(self._operation_validator.validate_bank_write(ctx)) + pool = await self._get_pool() + + op_uuid = uuid.UUID(operation_id) + + async with acquire_with_retry(pool) as conn: + row = await conn.fetchrow( + f"SELECT bank_id, status FROM {fq_table('async_operations')} WHERE operation_id = $1 AND bank_id = $2", + op_uuid, + bank_id, + ) + + if not row: + raise ValueError(f"Operation {operation_id} not found for bank {bank_id}") + + if row["status"] != "failed": + raise OperationValidationError( + f"Operation {operation_id} cannot be retried: status is '{row['status']}', expected 'failed'", + 409, + ) + + await conn.execute( + f""" + UPDATE {fq_table("async_operations")} + SET status = 'pending', + error_message = NULL, + completed_at = NULL, + next_retry_at = NULL, + worker_id = NULL, + claimed_at = NULL, + retry_count = 0, + updated_at = NOW() + WHERE operation_id = $1 + """, + op_uuid, + ) + + return { + "success": True, + "message": f"Operation {operation_id} queued for retry", + "operation_id": operation_id, + } + async def update_bank( self, bank_id: str, diff --git a/hindsight-clients/go/api/openapi.yaml b/hindsight-clients/go/api/openapi.yaml index d47d8447..4a153bf5 100644 --- a/hindsight-clients/go/api/openapi.yaml +++ b/hindsight-clients/go/api/openapi.yaml @@ -1771,6 +1771,51 @@ paths: summary: Get operation status tags: - Operations + /v1/default/banks/{bank_id}/operations/{operation_id}/retry: + post: + description: Re-queue a failed async operation so the worker picks it up again + operationId: retry_operation + parameters: + - explode: false + in: path + name: bank_id + required: true + schema: + title: Bank Id + type: string + style: simple + - explode: false + in: path + name: operation_id + required: true + schema: + title: Operation Id + type: string + style: simple + - explode: false + in: header + name: authorization + required: false + schema: + nullable: true + type: string + style: simple + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/RetryOperationResponse' + description: Successful Response + "422": + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + description: Validation Error + summary: Retry a failed async operation + tags: + - Operations /v1/default/banks/{bank_id}/profile: get: deprecated: true @@ -4748,6 +4793,27 @@ components: - items_count - success title: RetainResponse + RetryOperationResponse: + description: Response model for retry operation endpoint. + example: + message: Operation 550e8400-e29b-41d4-a716-446655440000 queued for retry + operation_id: 550e8400-e29b-41d4-a716-446655440000 + success: true + properties: + success: + title: Success + type: boolean + message: + title: Message + type: string + operation_id: + title: Operation Id + type: string + required: + - message + - operation_id + - success + title: RetryOperationResponse SourceFactsIncludeOptions: description: Options for including source facts for observation-type results. properties: diff --git a/hindsight-clients/go/api_operations.go b/hindsight-clients/go/api_operations.go index d0ed4f25..7ae8d262 100644 --- a/hindsight-clients/go/api_operations.go +++ b/hindsight-clients/go/api_operations.go @@ -442,3 +442,129 @@ func (a *OperationsAPIService) ListOperationsExecute(r ApiListOperationsRequest) return localVarReturnValue, localVarHTTPResponse, nil } + +type ApiRetryOperationRequest struct { + ctx context.Context + ApiService *OperationsAPIService + bankId string + operationId string + authorization *string +} + +func (r ApiRetryOperationRequest) Authorization(authorization string) ApiRetryOperationRequest { + r.authorization = &authorization + return r +} + +func (r ApiRetryOperationRequest) Execute() (*RetryOperationResponse, *http.Response, error) { + return r.ApiService.RetryOperationExecute(r) +} + +/* +RetryOperation Retry a failed async operation + +Re-queue a failed async operation so the worker picks it up again + + @param ctx context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). + @param bankId + @param operationId + @return ApiRetryOperationRequest +*/ +func (a *OperationsAPIService) RetryOperation(ctx context.Context, bankId string, operationId string) ApiRetryOperationRequest { + return ApiRetryOperationRequest{ + ApiService: a, + ctx: ctx, + bankId: bankId, + operationId: operationId, + } +} + +// Execute executes the request +// @return RetryOperationResponse +func (a *OperationsAPIService) RetryOperationExecute(r ApiRetryOperationRequest) (*RetryOperationResponse, *http.Response, error) { + var ( + localVarHTTPMethod = http.MethodPost + localVarPostBody interface{} + formFiles []formFile + localVarReturnValue *RetryOperationResponse + ) + + localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "OperationsAPIService.RetryOperation") + if err != nil { + return localVarReturnValue, nil, &GenericOpenAPIError{error: err.Error()} + } + + localVarPath := localBasePath + "/v1/default/banks/{bank_id}/operations/{operation_id}/retry" + localVarPath = strings.Replace(localVarPath, "{"+"bank_id"+"}", url.PathEscape(parameterValueToString(r.bankId, "bankId")), -1) + localVarPath = strings.Replace(localVarPath, "{"+"operation_id"+"}", url.PathEscape(parameterValueToString(r.operationId, "operationId")), -1) + + localVarHeaderParams := make(map[string]string) + localVarQueryParams := url.Values{} + localVarFormParams := url.Values{} + + // to determine the Content-Type header + localVarHTTPContentTypes := []string{} + + // set Content-Type header + localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) + if localVarHTTPContentType != "" { + localVarHeaderParams["Content-Type"] = localVarHTTPContentType + } + + // to determine the Accept header + localVarHTTPHeaderAccepts := []string{"application/json"} + + // set Accept header + localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) + if localVarHTTPHeaderAccept != "" { + localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept + } + if r.authorization != nil { + parameterAddToHeaderOrQuery(localVarHeaderParams, "authorization", r.authorization, "simple", "") + } + req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, formFiles) + if err != nil { + return localVarReturnValue, nil, err + } + + localVarHTTPResponse, err := a.client.callAPI(req) + if err != nil || localVarHTTPResponse == nil { + return localVarReturnValue, localVarHTTPResponse, err + } + + localVarBody, err := io.ReadAll(localVarHTTPResponse.Body) + localVarHTTPResponse.Body.Close() + localVarHTTPResponse.Body = io.NopCloser(bytes.NewBuffer(localVarBody)) + if err != nil { + return localVarReturnValue, localVarHTTPResponse, err + } + + if localVarHTTPResponse.StatusCode >= 300 { + newErr := &GenericOpenAPIError{ + body: localVarBody, + error: localVarHTTPResponse.Status, + } + if localVarHTTPResponse.StatusCode == 422 { + var v HTTPValidationError + err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) + if err != nil { + newErr.error = err.Error() + return localVarReturnValue, localVarHTTPResponse, newErr + } + newErr.error = formatErrorMessage(localVarHTTPResponse.Status, &v) + newErr.model = v + } + return localVarReturnValue, localVarHTTPResponse, newErr + } + + err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) + if err != nil { + newErr := &GenericOpenAPIError{ + body: localVarBody, + error: err.Error(), + } + return localVarReturnValue, localVarHTTPResponse, newErr + } + + return localVarReturnValue, localVarHTTPResponse, nil +} diff --git a/hindsight-clients/go/model_retry_operation_response.go b/hindsight-clients/go/model_retry_operation_response.go new file mode 100644 index 00000000..7205fb79 --- /dev/null +++ b/hindsight-clients/go/model_retry_operation_response.go @@ -0,0 +1,214 @@ +/* +Hindsight HTTP API + +HTTP API for Hindsight + +API version: 0.4.16 +*/ + +// Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT. + +package hindsight + +import ( + "encoding/json" + "bytes" + "fmt" +) + +// checks if the RetryOperationResponse type satisfies the MappedNullable interface at compile time +var _ MappedNullable = &RetryOperationResponse{} + +// RetryOperationResponse Response model for retry operation endpoint. +type RetryOperationResponse struct { + Success bool `json:"success"` + Message string `json:"message"` + OperationId string `json:"operation_id"` +} + +type _RetryOperationResponse RetryOperationResponse + +// NewRetryOperationResponse instantiates a new RetryOperationResponse object +// This constructor will assign default values to properties that have it defined, +// and makes sure properties required by API are set, but the set of arguments +// will change when the set of required properties is changed +func NewRetryOperationResponse(success bool, message string, operationId string) *RetryOperationResponse { + this := RetryOperationResponse{} + this.Success = success + this.Message = message + this.OperationId = operationId + return &this +} + +// NewRetryOperationResponseWithDefaults instantiates a new RetryOperationResponse object +// This constructor will only assign default values to properties that have it defined, +// but it doesn't guarantee that properties required by API are set +func NewRetryOperationResponseWithDefaults() *RetryOperationResponse { + this := RetryOperationResponse{} + return &this +} + +// GetSuccess returns the Success field value +func (o *RetryOperationResponse) GetSuccess() bool { + if o == nil { + var ret bool + return ret + } + + return o.Success +} + +// GetSuccessOk returns a tuple with the Success field value +// and a boolean to check if the value has been set. +func (o *RetryOperationResponse) GetSuccessOk() (*bool, bool) { + if o == nil { + return nil, false + } + return &o.Success, true +} + +// SetSuccess sets field value +func (o *RetryOperationResponse) SetSuccess(v bool) { + o.Success = v +} + +// GetMessage returns the Message field value +func (o *RetryOperationResponse) GetMessage() string { + if o == nil { + var ret string + return ret + } + + return o.Message +} + +// GetMessageOk returns a tuple with the Message field value +// and a boolean to check if the value has been set. +func (o *RetryOperationResponse) GetMessageOk() (*string, bool) { + if o == nil { + return nil, false + } + return &o.Message, true +} + +// SetMessage sets field value +func (o *RetryOperationResponse) SetMessage(v string) { + o.Message = v +} + +// GetOperationId returns the OperationId field value +func (o *RetryOperationResponse) GetOperationId() string { + if o == nil { + var ret string + return ret + } + + return o.OperationId +} + +// GetOperationIdOk returns a tuple with the OperationId field value +// and a boolean to check if the value has been set. +func (o *RetryOperationResponse) GetOperationIdOk() (*string, bool) { + if o == nil { + return nil, false + } + return &o.OperationId, true +} + +// SetOperationId sets field value +func (o *RetryOperationResponse) SetOperationId(v string) { + o.OperationId = v +} + +func (o RetryOperationResponse) MarshalJSON() ([]byte, error) { + toSerialize,err := o.ToMap() + if err != nil { + return []byte{}, err + } + return json.Marshal(toSerialize) +} + +func (o RetryOperationResponse) ToMap() (map[string]interface{}, error) { + toSerialize := map[string]interface{}{} + toSerialize["success"] = o.Success + toSerialize["message"] = o.Message + toSerialize["operation_id"] = o.OperationId + return toSerialize, nil +} + +func (o *RetryOperationResponse) UnmarshalJSON(data []byte) (err error) { + // This validates that all required properties are included in the JSON object + // by unmarshalling the object into a generic map with string keys and checking + // that every required field exists as a key in the generic map. + requiredProperties := []string{ + "success", + "message", + "operation_id", + } + + allProperties := make(map[string]interface{}) + + err = json.Unmarshal(data, &allProperties) + + if err != nil { + return err; + } + + for _, requiredProperty := range(requiredProperties) { + if _, exists := allProperties[requiredProperty]; !exists { + return fmt.Errorf("no value given for required property %v", requiredProperty) + } + } + + varRetryOperationResponse := _RetryOperationResponse{} + + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + err = decoder.Decode(&varRetryOperationResponse) + + if err != nil { + return err + } + + *o = RetryOperationResponse(varRetryOperationResponse) + + return err +} + +type NullableRetryOperationResponse struct { + value *RetryOperationResponse + isSet bool +} + +func (v NullableRetryOperationResponse) Get() *RetryOperationResponse { + return v.value +} + +func (v *NullableRetryOperationResponse) Set(val *RetryOperationResponse) { + v.value = val + v.isSet = true +} + +func (v NullableRetryOperationResponse) IsSet() bool { + return v.isSet +} + +func (v *NullableRetryOperationResponse) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableRetryOperationResponse(val *RetryOperationResponse) *NullableRetryOperationResponse { + return &NullableRetryOperationResponse{value: val, isSet: true} +} + +func (v NullableRetryOperationResponse) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableRetryOperationResponse) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} + + diff --git a/hindsight-clients/python/.openapi-generator/FILES b/hindsight-clients/python/.openapi-generator/FILES index dafa1cb0..2ae0c21a 100644 --- a/hindsight-clients/python/.openapi-generator/FILES +++ b/hindsight-clients/python/.openapi-generator/FILES @@ -81,6 +81,7 @@ hindsight_client_api/models/reflect_tool_call.py hindsight_client_api/models/reflect_trace.py hindsight_client_api/models/retain_request.py hindsight_client_api/models/retain_response.py +hindsight_client_api/models/retry_operation_response.py hindsight_client_api/models/source_facts_include_options.py hindsight_client_api/models/tag_item.py hindsight_client_api/models/timestamp.py diff --git a/hindsight-clients/python/hindsight_client_api/__init__.py b/hindsight-clients/python/hindsight_client_api/__init__.py index be624d22..12c15be5 100644 --- a/hindsight-clients/python/hindsight_client_api/__init__.py +++ b/hindsight-clients/python/hindsight_client_api/__init__.py @@ -106,6 +106,7 @@ from hindsight_client_api.models.reflect_tool_call import ReflectToolCall from hindsight_client_api.models.reflect_trace import ReflectTrace from hindsight_client_api.models.retain_request import RetainRequest from hindsight_client_api.models.retain_response import RetainResponse +from hindsight_client_api.models.retry_operation_response import RetryOperationResponse from hindsight_client_api.models.source_facts_include_options import SourceFactsIncludeOptions from hindsight_client_api.models.tag_item import TagItem from hindsight_client_api.models.timestamp import Timestamp diff --git a/hindsight-clients/python/hindsight_client_api/api/operations_api.py b/hindsight-clients/python/hindsight_client_api/api/operations_api.py index cef1dfbe..ac11da50 100644 --- a/hindsight-clients/python/hindsight_client_api/api/operations_api.py +++ b/hindsight-clients/python/hindsight_client_api/api/operations_api.py @@ -22,6 +22,7 @@ from typing_extensions import Annotated from hindsight_client_api.models.cancel_operation_response import CancelOperationResponse from hindsight_client_api.models.operation_status_response import OperationStatusResponse from hindsight_client_api.models.operations_list_response import OperationsListResponse +from hindsight_client_api.models.retry_operation_response import RetryOperationResponse from hindsight_client_api.api_client import ApiClient, RequestSerialized from hindsight_client_api.api_response import ApiResponse @@ -971,3 +972,296 @@ class OperationsApi: ) + + + @validate_call + async def retry_operation( + self, + bank_id: StrictStr, + operation_id: StrictStr, + authorization: Optional[StrictStr] = None, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> RetryOperationResponse: + """Retry a failed async operation + + Re-queue a failed async operation so the worker picks it up again + + :param bank_id: (required) + :type bank_id: str + :param operation_id: (required) + :type operation_id: str + :param authorization: + :type authorization: str + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._retry_operation_serialize( + bank_id=bank_id, + operation_id=operation_id, + authorization=authorization, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "RetryOperationResponse", + '422': "HTTPValidationError", + } + response_data = await self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + await response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ).data + + + @validate_call + async def retry_operation_with_http_info( + self, + bank_id: StrictStr, + operation_id: StrictStr, + authorization: Optional[StrictStr] = None, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> ApiResponse[RetryOperationResponse]: + """Retry a failed async operation + + Re-queue a failed async operation so the worker picks it up again + + :param bank_id: (required) + :type bank_id: str + :param operation_id: (required) + :type operation_id: str + :param authorization: + :type authorization: str + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._retry_operation_serialize( + bank_id=bank_id, + operation_id=operation_id, + authorization=authorization, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "RetryOperationResponse", + '422': "HTTPValidationError", + } + response_data = await self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + await response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ) + + + @validate_call + async def retry_operation_without_preload_content( + self, + bank_id: StrictStr, + operation_id: StrictStr, + authorization: Optional[StrictStr] = None, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> RESTResponseType: + """Retry a failed async operation + + Re-queue a failed async operation so the worker picks it up again + + :param bank_id: (required) + :type bank_id: str + :param operation_id: (required) + :type operation_id: str + :param authorization: + :type authorization: str + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._retry_operation_serialize( + bank_id=bank_id, + operation_id=operation_id, + authorization=authorization, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "RetryOperationResponse", + '422': "HTTPValidationError", + } + response_data = await self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + return response_data.response + + + def _retry_operation_serialize( + self, + bank_id, + operation_id, + authorization, + _request_auth, + _content_type, + _headers, + _host_index, + ) -> RequestSerialized: + + _host = None + + _collection_formats: Dict[str, str] = { + } + + _path_params: Dict[str, str] = {} + _query_params: List[Tuple[str, str]] = [] + _header_params: Dict[str, Optional[str]] = _headers or {} + _form_params: List[Tuple[str, str]] = [] + _files: Dict[ + str, Union[str, bytes, List[str], List[bytes], List[Tuple[str, bytes]]] + ] = {} + _body_params: Optional[bytes] = None + + # process the path parameters + if bank_id is not None: + _path_params['bank_id'] = bank_id + if operation_id is not None: + _path_params['operation_id'] = operation_id + # process the query parameters + # process the header parameters + if authorization is not None: + _header_params['authorization'] = authorization + # process the form parameters + # process the body parameter + + + # set the HTTP header `Accept` + if 'Accept' not in _header_params: + _header_params['Accept'] = self.api_client.select_header_accept( + [ + 'application/json' + ] + ) + + + # authentication setting + _auth_settings: List[str] = [ + ] + + return self.api_client.param_serialize( + method='POST', + resource_path='/v1/default/banks/{bank_id}/operations/{operation_id}/retry', + path_params=_path_params, + query_params=_query_params, + header_params=_header_params, + body=_body_params, + post_params=_form_params, + files=_files, + auth_settings=_auth_settings, + collection_formats=_collection_formats, + _host=_host, + _request_auth=_request_auth + ) + + diff --git a/hindsight-clients/python/hindsight_client_api/models/__init__.py b/hindsight-clients/python/hindsight_client_api/models/__init__.py index 69fde0be..29c3ad1d 100644 --- a/hindsight-clients/python/hindsight_client_api/models/__init__.py +++ b/hindsight-clients/python/hindsight_client_api/models/__init__.py @@ -80,6 +80,7 @@ from hindsight_client_api.models.reflect_tool_call import ReflectToolCall from hindsight_client_api.models.reflect_trace import ReflectTrace from hindsight_client_api.models.retain_request import RetainRequest from hindsight_client_api.models.retain_response import RetainResponse +from hindsight_client_api.models.retry_operation_response import RetryOperationResponse from hindsight_client_api.models.source_facts_include_options import SourceFactsIncludeOptions from hindsight_client_api.models.tag_item import TagItem from hindsight_client_api.models.timestamp import Timestamp diff --git a/hindsight-clients/python/hindsight_client_api/models/retry_operation_response.py b/hindsight-clients/python/hindsight_client_api/models/retry_operation_response.py new file mode 100644 index 00000000..3bfb4b8c --- /dev/null +++ b/hindsight-clients/python/hindsight_client_api/models/retry_operation_response.py @@ -0,0 +1,91 @@ +# coding: utf-8 + +""" + Hindsight HTTP API + + HTTP API for Hindsight + + The version of the OpenAPI document: 0.4.16 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, StrictBool, StrictStr +from typing import Any, ClassVar, Dict, List +from typing import Optional, Set +from typing_extensions import Self + +class RetryOperationResponse(BaseModel): + """ + Response model for retry operation endpoint. + """ # noqa: E501 + success: StrictBool + message: StrictStr + operation_id: StrictStr + __properties: ClassVar[List[str]] = ["success", "message", "operation_id"] + + model_config = ConfigDict( + populate_by_name=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of RetryOperationResponse from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of RetryOperationResponse from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "success": obj.get("success"), + "message": obj.get("message"), + "operation_id": obj.get("operation_id") + }) + return _obj + + diff --git a/hindsight-clients/typescript/generated/sdk.gen.ts b/hindsight-clients/typescript/generated/sdk.gen.ts index c64da0f8..483793ed 100644 --- a/hindsight-clients/typescript/generated/sdk.gen.ts +++ b/hindsight-clients/typescript/generated/sdk.gen.ts @@ -146,6 +146,9 @@ import type { RetainMemoriesData, RetainMemoriesErrors, RetainMemoriesResponses, + RetryOperationData, + RetryOperationErrors, + RetryOperationResponses, TriggerConsolidationData, TriggerConsolidationErrors, TriggerConsolidationResponses, @@ -783,6 +786,23 @@ export const getOperationStatus = ( ...options, }); +/** + * Retry a failed async operation + * + * Re-queue a failed async operation so the worker picks it up again + */ +export const retryOperation = ( + options: Options, +) => + (options.client ?? client).post< + RetryOperationResponses, + RetryOperationErrors, + ThrowOnError + >({ + url: "/v1/default/banks/{bank_id}/operations/{operation_id}/retry", + ...options, + }); + /** * Get memory bank profile * diff --git a/hindsight-clients/typescript/generated/types.gen.ts b/hindsight-clients/typescript/generated/types.gen.ts index 3fbec174..03bcbd82 100644 --- a/hindsight-clients/typescript/generated/types.gen.ts +++ b/hindsight-clients/typescript/generated/types.gen.ts @@ -1951,6 +1951,26 @@ export type RetainResponse = { usage?: TokenUsage | null; }; +/** + * RetryOperationResponse + * + * Response model for retry operation endpoint. + */ +export type RetryOperationResponse = { + /** + * Success + */ + success: boolean; + /** + * Message + */ + message: string; + /** + * Operation Id + */ + operation_id: string; +}; + /** * SourceFactsIncludeOptions * @@ -3841,6 +3861,48 @@ export type GetOperationStatusResponses = { export type GetOperationStatusResponse = GetOperationStatusResponses[keyof GetOperationStatusResponses]; +export type RetryOperationData = { + body?: never; + headers?: { + /** + * Authorization + */ + authorization?: string | null; + }; + path: { + /** + * Bank Id + */ + bank_id: string; + /** + * Operation Id + */ + operation_id: string; + }; + query?: never; + url: "/v1/default/banks/{bank_id}/operations/{operation_id}/retry"; +}; + +export type RetryOperationErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type RetryOperationError = + RetryOperationErrors[keyof RetryOperationErrors]; + +export type RetryOperationResponses = { + /** + * Successful Response + */ + 200: RetryOperationResponse; +}; + +export type RetryOperationResponse2 = + RetryOperationResponses[keyof RetryOperationResponses]; + export type GetBankProfileData = { body?: never; headers?: { diff --git a/hindsight-control-plane/src/app/api/banks/[bankId]/operations/[operationId]/route.ts b/hindsight-control-plane/src/app/api/banks/[bankId]/operations/[operationId]/route.ts index 50f3bb4b..ab5b1353 100644 --- a/hindsight-control-plane/src/app/api/banks/[bankId]/operations/[operationId]/route.ts +++ b/hindsight-control-plane/src/app/api/banks/[bankId]/operations/[operationId]/route.ts @@ -1,5 +1,5 @@ import { NextResponse } from "next/server"; -import { sdk, lowLevelClient } from "@/lib/hindsight-client"; +import { sdk, lowLevelClient, DATAPLANE_URL, getDataplaneHeaders } from "@/lib/hindsight-client"; export async function GET( request: Request, @@ -32,3 +32,40 @@ export async function GET( return NextResponse.json({ error: "Failed to get operation status" }, { status: 500 }); } } + +export async function POST( + request: Request, + { params }: { params: Promise<{ bankId: string; operationId: string }> } +) { + try { + const { bankId, operationId } = await params; + + if (!bankId) { + return NextResponse.json({ error: "bank_id is required" }, { status: 400 }); + } + + if (!operationId) { + return NextResponse.json({ error: "operation_id is required" }, { status: 400 }); + } + + const url = `${DATAPLANE_URL}/v1/default/banks/${bankId}/operations/${operationId}/retry`; + const response = await fetch(url, { + method: "POST", + headers: getDataplaneHeaders({ "Content-Type": "application/json" }), + }); + + const data = await response.json(); + + if (!response.ok) { + return NextResponse.json( + { error: data.detail || "Failed to retry operation" }, + { status: response.status } + ); + } + + return NextResponse.json(data, { status: 200 }); + } catch (error) { + console.error("Error retrying operation:", error); + return NextResponse.json({ error: "Failed to retry operation" }, { status: 500 }); + } +} diff --git a/hindsight-control-plane/src/components/bank-operations-view.tsx b/hindsight-control-plane/src/components/bank-operations-view.tsx index 7023453f..31076d13 100644 --- a/hindsight-control-plane/src/components/bank-operations-view.tsx +++ b/hindsight-control-plane/src/components/bank-operations-view.tsx @@ -26,7 +26,7 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; -import { RefreshCw, Clock, AlertCircle, CheckCircle, Loader2, X } from "lucide-react"; +import { RefreshCw, Clock, AlertCircle, CheckCircle, Loader2, X, RotateCcw } from "lucide-react"; interface Operation { id: string; @@ -96,6 +96,7 @@ export function BankOperationsView() { const [limit] = useState(10); const [offset, setOffset] = useState(0); const [cancellingOpId, setCancellingOpId] = useState(null); + const [retryingOpId, setRetryingOpId] = useState(null); const [loading, setLoading] = useState(false); const [selectedOperation, setSelectedOperation] = useState(null); const [dialogOpen, setDialogOpen] = useState(false); @@ -159,6 +160,20 @@ export function BankOperationsView() { } }; + const handleRetryOperation = async (operationId: string) => { + if (!currentBank) return; + + setRetryingOpId(operationId); + try { + await client.retryOperation(currentBank, operationId); + await loadOperations(); + } catch (error) { + // Error toast is shown automatically by the API client interceptor + } finally { + setRetryingOpId(null); + } + }; + const handleOperationClick = async (operationId: string) => { if (!currentBank) return; @@ -324,6 +339,25 @@ export function BankOperationsView() { {cancellingOpId === op.id ? "" : "Cancel"} )} + {op.status === "failed" && ( + + )} ))} diff --git a/hindsight-control-plane/src/lib/api.ts b/hindsight-control-plane/src/lib/api.ts index b22be686..ece91de4 100644 --- a/hindsight-control-plane/src/lib/api.ts +++ b/hindsight-control-plane/src/lib/api.ts @@ -286,6 +286,19 @@ export class ControlPlaneClient { }); } + /** + * Retry a failed operation + */ + async retryOperation(bankId: string, operationId: string) { + return this.fetchApi<{ + success: boolean; + message: string; + operation_id: string; + }>(`/api/banks/${bankId}/operations/${operationId}`, { + method: "POST", + }); + } + /** * List entities */ diff --git a/hindsight-docs/docs/developer/api/operations.md b/hindsight-docs/docs/developer/api/operations.md index 76540b08..5a4dc950 100644 --- a/hindsight-docs/docs/developer/api/operations.md +++ b/hindsight-docs/docs/developer/api/operations.md @@ -88,6 +88,33 @@ Response: | `completed` | Operation finished successfully | | `failed` | Operation failed (check `error_message` for details) | +## Managing Operations + +### Cancel a pending operation + +```bash +curl -X DELETE "http://localhost:8000/v1/default/banks/my-bank/operations/550e8400-e29b-41d4-a716-446655440000" +``` + +### Retry a failed operation + +If an operation fails, you can manually re-queue it for execution: + +```bash +curl -X POST "http://localhost:8000/v1/default/banks/my-bank/operations/550e8400-e29b-41d4-a716-446655440000/retry" +``` + +Response: +```json +{ + "success": true, + "message": "Operation 550e8400-e29b-41d4-a716-446655440000 queued for retry", + "operation_id": "550e8400-e29b-41d4-a716-446655440000" +} +``` + +The operation status resets to `pending` and the worker picks it up again. Returns `409` if the operation is not in `failed` state. + ## Next Steps - [**Documents**](./documents) — Track document sources diff --git a/hindsight-docs/static/openapi.json b/hindsight-docs/static/openapi.json index 904f53b7..4eacda1a 100644 --- a/hindsight-docs/static/openapi.json +++ b/hindsight-docs/static/openapi.json @@ -2607,6 +2607,74 @@ } } }, + "/v1/default/banks/{bank_id}/operations/{operation_id}/retry": { + "post": { + "tags": [ + "Operations" + ], + "summary": "Retry a failed async operation", + "description": "Re-queue a failed async operation so the worker picks it up again", + "operationId": "retry_operation", + "parameters": [ + { + "name": "bank_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Bank Id" + } + }, + { + "name": "operation_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Operation Id" + } + }, + { + "name": "authorization", + "in": "header", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RetryOperationResponse" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/v1/default/banks/{bank_id}/profile": { "get": { "tags": [ @@ -7431,6 +7499,35 @@ } } }, + "RetryOperationResponse": { + "properties": { + "success": { + "type": "boolean", + "title": "Success" + }, + "message": { + "type": "string", + "title": "Message" + }, + "operation_id": { + "type": "string", + "title": "Operation Id" + } + }, + "type": "object", + "required": [ + "success", + "message", + "operation_id" + ], + "title": "RetryOperationResponse", + "description": "Response model for retry operation endpoint.", + "example": { + "message": "Operation 550e8400-e29b-41d4-a716-446655440000 queued for retry", + "operation_id": "550e8400-e29b-41d4-a716-446655440000", + "success": true + } + }, "SourceFactsIncludeOptions": { "properties": { "max_tokens": {