From 646dea89d9567adbf927c714262d611328c18d76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 10 Nov 2025 15:45:30 +0100 Subject: [PATCH] reorder files --- .sesskey | 1 + .../benchmarks-results}/locomo/73.5.json | 0 .../longmemeval/final.json | 0 benchmarks/locomo/locomo_benchmark.py | 7 +- .../longmemeval/longmemeval_benchmark.py | 7 +- benchmarks/pyproject.toml | 23 ++++++ control-plane/.dockerignore | 12 +++ control-plane/.env.local.example | 4 +- control-plane/Dockerfile | 39 ++++++++++ control-plane/README.md | 2 +- control-plane/build-docker.sh | 74 +++++++++++++++++++ control-plane/next.config.ts | 2 +- control-plane/src/app/api/agents/route.ts | 2 +- .../app/api/documents/[documentId]/route.ts | 2 +- control-plane/src/app/api/documents/route.ts | 2 +- control-plane/src/app/api/graph/route.ts | 2 +- control-plane/src/app/api/list/route.ts | 2 +- .../src/app/api/memories/batch/route.ts | 2 +- .../src/app/api/memories/batch_async/route.ts | 2 +- .../src/app/api/operations/[agentId]/route.ts | 2 +- control-plane/src/app/api/search/route.ts | 2 +- .../src/app/api/stats/[agentId]/route.ts | 2 +- control-plane/src/app/api/think/route.ts | 2 +- control-plane/src/lib/api.ts | 2 +- {docker => local-db}/.gitignore | 0 {docker => local-db}/docker-compose.yml | 0 {docker => local-db}/init-extensions.sql | 0 memora-dev/memora_dev/__init__.py | 1 + .../memora_dev/generate_openapi.py | 3 - memora-dev/pyproject.toml | 18 +++++ README.md => memora/README.md | 0 alembic.ini => memora/alembic.ini | 0 {alembic => memora/alembic}/README | 0 {alembic => memora/alembic}/env.py | 11 ++- {alembic => memora/alembic}/script.py.mako | 0 ...b9079_fix_memory_links_entity_id_to_be_.py | 0 ...0e96398aae9e_add_async_operations_table.py | 0 .../1a35a4fa1950_add_bm25_fulltext_search.py | 0 ...dd_status_and_error_to_async_operations.py | 0 .../8c55f5602451_remove_entity_type_column.py | 0 .../versions/af0413383b3e_initial_schema.py | 0 memora/{ => memora}/__init__.py | 0 memora/{ => memora}/api.py | 0 memora/{ => memora}/cross_encoder.py | 0 memora/{ => memora}/embeddings.py | 0 memora/{ => memora}/entity_resolver.py | 0 memora/{ => memora}/fact_extraction.py | 0 memora/{ => memora}/llm_wrapper.py | 0 memora/{ => memora}/models.py | 0 memora/{ => memora}/operations/__init__.py | 0 .../operations/batch_operations.py | 0 .../operations/embedding_operations.py | 0 .../operations/link_operations.py | 0 .../operations/think_operations.py | 0 memora/{ => memora}/remote_client.py | 0 memora/{ => memora}/search/__init__.py | 0 memora/{ => memora}/search/reranking.py | 0 memora/{ => memora}/search/retrieval.py | 0 .../search/temporal_extraction.py | 0 memora/{ => memora}/search_helpers.py | 0 memora/{ => memora}/search_trace.py | 0 memora/{ => memora}/search_tracer.py | 0 memora/{ => memora}/task_backend.py | 0 .../{ => memora}/temporal_semantic_memory.py | 0 memora/{ => memora}/utils.py | 0 memora/{ => memora}/web/__init__.py | 0 memora/{ => memora}/web/server.py | 0 memora/pyproject.toml | 43 +++++++++++ {tests => memora/tests}/__init__.py | 0 {tests => memora/tests}/conftest.py | 3 +- {tests => memora/tests}/fixtures/README.md | 0 .../fixtures/locomo_conversation_sample.json | 0 .../tests}/test_batch_chunking.py | 0 {tests => memora/tests}/test_chunking.py | 0 .../tests}/test_document_tracking.py | 0 .../tests}/test_performance_tuning.py | 0 {tests => memora/tests}/test_search_trace.py | 0 .../tests}/test_temporal_extraction.py | 0 {tests => memora/tests}/test_think.py | 0 openapi.json | 46 ++++++------ pyproject.toml | 48 +----------- scripts/erase-local-db.sh | 2 +- scripts/migrate-db.sh | 2 +- scripts/start-local-db.sh | 4 +- uv.lock | 47 ++++++++++-- 85 files changed, 312 insertions(+), 111 deletions(-) create mode 100644 .sesskey rename {benchmarks-results => benchmarks/benchmarks-results}/locomo/73.5.json (100%) rename {benchmarks-results => benchmarks/benchmarks-results}/longmemeval/final.json (100%) create mode 100644 benchmarks/pyproject.toml create mode 100644 control-plane/.dockerignore create mode 100644 control-plane/Dockerfile create mode 100755 control-plane/build-docker.sh rename {docker => local-db}/.gitignore (100%) rename {docker => local-db}/docker-compose.yml (100%) rename {docker => local-db}/init-extensions.sql (100%) create mode 100644 memora-dev/memora_dev/__init__.py rename generate_openapi.py => memora-dev/memora_dev/generate_openapi.py (94%) create mode 100644 memora-dev/pyproject.toml rename README.md => memora/README.md (100%) rename alembic.ini => memora/alembic.ini (100%) rename {alembic => memora/alembic}/README (100%) rename {alembic => memora/alembic}/env.py (92%) rename {alembic => memora/alembic}/script.py.mako (100%) rename {alembic => memora/alembic}/versions/01f989db9079_fix_memory_links_entity_id_to_be_.py (100%) rename {alembic => memora/alembic}/versions/0e96398aae9e_add_async_operations_table.py (100%) rename {alembic => memora/alembic}/versions/1a35a4fa1950_add_bm25_fulltext_search.py (100%) rename {alembic => memora/alembic}/versions/2a76a5bc2f09_add_status_and_error_to_async_operations.py (100%) rename {alembic => memora/alembic}/versions/8c55f5602451_remove_entity_type_column.py (100%) rename {alembic => memora/alembic}/versions/af0413383b3e_initial_schema.py (100%) rename memora/{ => memora}/__init__.py (100%) rename memora/{ => memora}/api.py (100%) rename memora/{ => memora}/cross_encoder.py (100%) rename memora/{ => memora}/embeddings.py (100%) rename memora/{ => memora}/entity_resolver.py (100%) rename memora/{ => memora}/fact_extraction.py (100%) rename memora/{ => memora}/llm_wrapper.py (100%) rename memora/{ => memora}/models.py (100%) rename memora/{ => memora}/operations/__init__.py (100%) rename memora/{ => memora}/operations/batch_operations.py (100%) rename memora/{ => memora}/operations/embedding_operations.py (100%) rename memora/{ => memora}/operations/link_operations.py (100%) rename memora/{ => memora}/operations/think_operations.py (100%) rename memora/{ => memora}/remote_client.py (100%) rename memora/{ => memora}/search/__init__.py (100%) rename memora/{ => memora}/search/reranking.py (100%) rename memora/{ => memora}/search/retrieval.py (100%) rename memora/{ => memora}/search/temporal_extraction.py (100%) rename memora/{ => memora}/search_helpers.py (100%) rename memora/{ => memora}/search_trace.py (100%) rename memora/{ => memora}/search_tracer.py (100%) rename memora/{ => memora}/task_backend.py (100%) rename memora/{ => memora}/temporal_semantic_memory.py (100%) rename memora/{ => memora}/utils.py (100%) rename memora/{ => memora}/web/__init__.py (100%) rename memora/{ => memora}/web/server.py (100%) create mode 100644 memora/pyproject.toml rename {tests => memora/tests}/__init__.py (100%) rename {tests => memora/tests}/conftest.py (95%) rename {tests => memora/tests}/fixtures/README.md (100%) rename {tests => memora/tests}/fixtures/locomo_conversation_sample.json (100%) rename {tests => memora/tests}/test_batch_chunking.py (100%) rename {tests => memora/tests}/test_chunking.py (100%) rename {tests => memora/tests}/test_document_tracking.py (100%) rename {tests => memora/tests}/test_performance_tuning.py (100%) rename {tests => memora/tests}/test_search_trace.py (100%) rename {tests => memora/tests}/test_temporal_extraction.py (100%) rename {tests => memora/tests}/test_think.py (100%) diff --git a/.sesskey b/.sesskey new file mode 100644 index 00000000..755d65de --- /dev/null +++ b/.sesskey @@ -0,0 +1 @@ +49ed5f3e-d51f-4fbe-abdb-c909287df6a0 \ No newline at end of file diff --git a/benchmarks-results/locomo/73.5.json b/benchmarks/benchmarks-results/locomo/73.5.json similarity index 100% rename from benchmarks-results/locomo/73.5.json rename to benchmarks/benchmarks-results/locomo/73.5.json diff --git a/benchmarks-results/longmemeval/final.json b/benchmarks/benchmarks-results/longmemeval/final.json similarity index 100% rename from benchmarks-results/longmemeval/final.json rename to benchmarks/benchmarks-results/longmemeval/final.json diff --git a/benchmarks/locomo/locomo_benchmark.py b/benchmarks/locomo/locomo_benchmark.py index 097e1359..bb7f0141 100644 --- a/benchmarks/locomo/locomo_benchmark.py +++ b/benchmarks/locomo/locomo_benchmark.py @@ -9,9 +9,6 @@ from pathlib import Path from benchmarks.common.benchmark_runner import BenchmarkRunner from memora import TemporalSemanticMemory -# Add parent directory to path for imports -sys.path.insert(0, str(Path(__file__).parent.parent.parent)) - import json from datetime import datetime, timezone from typing import List, Dict, Any, Tuple, Optional @@ -20,9 +17,7 @@ import pydantic from openai import AsyncOpenAI import os -# Import common framework -sys.path.insert(0, str(Path(__file__).parent.parent)) -from common.benchmark_runner import BenchmarkDataset, LLMAnswerGenerator, LLMAnswerEvaluator +from benchmarks.common.benchmark_runner import BenchmarkDataset, LLMAnswerGenerator, LLMAnswerEvaluator from memora.llm_wrapper import LLMConfig class LoComoDataset(BenchmarkDataset): diff --git a/benchmarks/longmemeval/longmemeval_benchmark.py b/benchmarks/longmemeval/longmemeval_benchmark.py index 3d228280..3ef7fbf9 100644 --- a/benchmarks/longmemeval/longmemeval_benchmark.py +++ b/benchmarks/longmemeval/longmemeval_benchmark.py @@ -9,9 +9,6 @@ from pathlib import Path from benchmarks.common.benchmark_runner import BenchmarkRunner from memora import TemporalSemanticMemory -# Add parent directory to path for imports -sys.path.insert(0, str(Path(__file__).parent.parent.parent)) - import json from datetime import datetime, timezone from typing import List, Dict, Any, Tuple, Optional @@ -20,9 +17,7 @@ import pydantic from openai import AsyncOpenAI import os -# Import common framework -sys.path.insert(0, str(Path(__file__).parent.parent)) -from common.benchmark_runner import BenchmarkDataset, LLMAnswerGenerator, LLMAnswerEvaluator +from benchmarks.common.benchmark_runner import BenchmarkDataset, LLMAnswerGenerator, LLMAnswerEvaluator from memora.llm_wrapper import LLMConfig diff --git a/benchmarks/pyproject.toml b/benchmarks/pyproject.toml new file mode 100644 index 00000000..732b6c8d --- /dev/null +++ b/benchmarks/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "benchmarks" +version = "0.1.0" +description = "Benchmarks for Memora memory system" +requires-python = ">=3.11" +dependencies = [ + "memora", + "python-fasthtml>=0.12.33", + "streamlit>=1.51.0", + "openai>=1.0.0", + "rich>=13.0.0", + "pydantic>=2.0.0", +] + +[tool.hatch.build.targets.wheel] +packages = ["benchmarks"] + +[tool.uv.sources] +memora = { workspace = true } diff --git a/control-plane/.dockerignore b/control-plane/.dockerignore new file mode 100644 index 00000000..bedce897 --- /dev/null +++ b/control-plane/.dockerignore @@ -0,0 +1,12 @@ +.next +node_modules +npm-debug.log +.git +.gitignore +.env.local +.env*.local +README.md +Dockerfile +.dockerignore +build-docker.sh +docker-compose.yml diff --git a/control-plane/.env.local.example b/control-plane/.env.local.example index b36d3e5f..3a8e2a34 100644 --- a/control-plane/.env.local.example +++ b/control-plane/.env.local.example @@ -1,3 +1,3 @@ # Dataplane API Configuration -# URL of the Python FastAPI dataplane server -NEXT_PUBLIC_DATAPLANE_API_URL=http://localhost:8080 +# URL of the Python FastAPI dataplane server (server-side only, not exposed to browser) +DATAPLANE_API_URL=http://localhost:8080 diff --git a/control-plane/Dockerfile b/control-plane/Dockerfile new file mode 100644 index 00000000..55309122 --- /dev/null +++ b/control-plane/Dockerfile @@ -0,0 +1,39 @@ +FROM node:20-alpine AS base + +# Install dependencies only when needed +FROM base AS deps +RUN apk add --no-cache libc6-compat +WORKDIR /app + +COPY package.json package-lock.json ./ +RUN npm ci + +# Rebuild the source code only when needed +FROM base AS builder +WORKDIR /app +COPY --from=deps /app/node_modules ./node_modules +COPY . . + +RUN npm run build + +# Production image, copy all the files and run next +FROM base AS runner +WORKDIR /app + +ENV NODE_ENV=production + +RUN addgroup --system --gid 1001 nodejs +RUN adduser --system --uid 1001 nextjs + +# Automatically leverage output traces to reduce image size +COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ +COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static + +USER nextjs + +EXPOSE 3000 + +ENV PORT=3000 +ENV HOSTNAME="0.0.0.0" + +CMD ["node", "server.js"] diff --git a/control-plane/README.md b/control-plane/README.md index bd432af6..28805f94 100644 --- a/control-plane/README.md +++ b/control-plane/README.md @@ -84,7 +84,7 @@ cp .env.local.example .env.local Edit `.env.local`: ```env -NEXT_PUBLIC_DATAPLANE_API_URL=http://localhost:8080 +DATAPLANE_API_URL=http://localhost:8080 ``` ### Development diff --git a/control-plane/build-docker.sh b/control-plane/build-docker.sh new file mode 100755 index 00000000..c8a6a13f --- /dev/null +++ b/control-plane/build-docker.sh @@ -0,0 +1,74 @@ +#!/bin/bash +set -e + +# Default values +IMAGE_NAME="memora-control-plane" +IMAGE_TAG="latest" +REGISTRY="" + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --name) + IMAGE_NAME="$2" + shift 2 + ;; + --tag) + IMAGE_TAG="$2" + shift 2 + ;; + --registry) + REGISTRY="$2" + shift 2 + ;; + --help) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --name NAME Docker image name (default: control-plane)" + echo " --tag TAG Docker image tag (default: latest)" + echo " --registry REG Docker registry URL (optional)" + echo " --help Show this help message" + echo "" + echo "Example:" + echo " $0 --name myapp --tag v1.0.0" + echo " $0 --registry docker.io/myuser --name control-plane --tag v1.0.0" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Construct full image name +if [ -n "$REGISTRY" ]; then + FULL_IMAGE_NAME="${REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}" +else + FULL_IMAGE_NAME="${IMAGE_NAME}:${IMAGE_TAG}" +fi + +echo "Building Docker image: ${FULL_IMAGE_NAME}" +echo "========================================" + +# Build the Docker image +docker build -t "${FULL_IMAGE_NAME}" . + +echo "" +echo "Build completed successfully!" +echo "Image: ${FULL_IMAGE_NAME}" +echo "" +echo "To run the container:" +echo " docker run -p 3000:3000 \\" +echo " -e DATAPLANE_API_URL=http://your-api-url:8080 \\" +echo " ${FULL_IMAGE_NAME}" +echo "" +echo "Or with an env file:" +echo " docker run -p 3000:3000 --env-file .env.local ${FULL_IMAGE_NAME}" +echo "" +echo "To push to registry (if registry specified):" +if [ -n "$REGISTRY" ]; then + echo " docker push ${FULL_IMAGE_NAME}" +fi diff --git a/control-plane/next.config.ts b/control-plane/next.config.ts index e9ffa308..225e4952 100644 --- a/control-plane/next.config.ts +++ b/control-plane/next.config.ts @@ -1,7 +1,7 @@ import type { NextConfig } from "next"; const nextConfig: NextConfig = { - /* config options here */ + output: 'standalone', }; export default nextConfig; diff --git a/control-plane/src/app/api/agents/route.ts b/control-plane/src/app/api/agents/route.ts index c14957a7..31272008 100644 --- a/control-plane/src/app/api/agents/route.ts +++ b/control-plane/src/app/api/agents/route.ts @@ -1,6 +1,6 @@ import { NextResponse } from 'next/server'; -const DATAPLANE_URL = process.env.NEXT_PUBLIC_DATAPLANE_API_URL || 'http://localhost:8080'; +const DATAPLANE_URL = process.env.DATAPLANE_API_URL || 'http://localhost:8080'; export async function GET() { try { diff --git a/control-plane/src/app/api/documents/[documentId]/route.ts b/control-plane/src/app/api/documents/[documentId]/route.ts index b27cfcb0..b0bd12be 100644 --- a/control-plane/src/app/api/documents/[documentId]/route.ts +++ b/control-plane/src/app/api/documents/[documentId]/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; -const DATAPLANE_URL = process.env.NEXT_PUBLIC_DATAPLANE_API_URL || 'http://localhost:8080'; +const DATAPLANE_URL = process.env.DATAPLANE_API_URL || 'http://localhost:8080'; export async function GET( request: NextRequest, diff --git a/control-plane/src/app/api/documents/route.ts b/control-plane/src/app/api/documents/route.ts index 5b0b1588..38e2e864 100644 --- a/control-plane/src/app/api/documents/route.ts +++ b/control-plane/src/app/api/documents/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; -const DATAPLANE_URL = process.env.NEXT_PUBLIC_DATAPLANE_API_URL || 'http://localhost:8080'; +const DATAPLANE_URL = process.env.DATAPLANE_API_URL || 'http://localhost:8080'; export async function GET(request: NextRequest) { try { diff --git a/control-plane/src/app/api/graph/route.ts b/control-plane/src/app/api/graph/route.ts index 9d6aa0fb..d81f6918 100644 --- a/control-plane/src/app/api/graph/route.ts +++ b/control-plane/src/app/api/graph/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; -const DATAPLANE_URL = process.env.NEXT_PUBLIC_DATAPLANE_API_URL || 'http://localhost:8080'; +const DATAPLANE_URL = process.env.DATAPLANE_API_URL || 'http://localhost:8080'; export async function GET(request: NextRequest) { try { diff --git a/control-plane/src/app/api/list/route.ts b/control-plane/src/app/api/list/route.ts index 806ac8e8..76c119ca 100644 --- a/control-plane/src/app/api/list/route.ts +++ b/control-plane/src/app/api/list/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; -const DATAPLANE_URL = process.env.NEXT_PUBLIC_DATAPLANE_API_URL || 'http://localhost:8080'; +const DATAPLANE_URL = process.env.DATAPLANE_API_URL || 'http://localhost:8080'; export async function GET(request: NextRequest) { try { diff --git a/control-plane/src/app/api/memories/batch/route.ts b/control-plane/src/app/api/memories/batch/route.ts index cff85492..1abe01d1 100644 --- a/control-plane/src/app/api/memories/batch/route.ts +++ b/control-plane/src/app/api/memories/batch/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; -const DATAPLANE_URL = process.env.NEXT_PUBLIC_DATAPLANE_API_URL || 'http://localhost:8080'; +const DATAPLANE_URL = process.env.DATAPLANE_API_URL || 'http://localhost:8080'; export async function POST(request: NextRequest) { try { diff --git a/control-plane/src/app/api/memories/batch_async/route.ts b/control-plane/src/app/api/memories/batch_async/route.ts index fec12f2f..7a3b6ae6 100644 --- a/control-plane/src/app/api/memories/batch_async/route.ts +++ b/control-plane/src/app/api/memories/batch_async/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; -const DATAPLANE_URL = process.env.NEXT_PUBLIC_DATAPLANE_API_URL || 'http://localhost:8080'; +const DATAPLANE_URL = process.env.DATAPLANE_API_URL || 'http://localhost:8080'; export async function POST(request: NextRequest) { try { diff --git a/control-plane/src/app/api/operations/[agentId]/route.ts b/control-plane/src/app/api/operations/[agentId]/route.ts index bd33885d..86782921 100644 --- a/control-plane/src/app/api/operations/[agentId]/route.ts +++ b/control-plane/src/app/api/operations/[agentId]/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; -const DATAPLANE_URL = process.env.NEXT_PUBLIC_DATAPLANE_API_URL || 'http://localhost:8080'; +const DATAPLANE_URL = process.env.DATAPLANE_API_URL || 'http://localhost:8080'; export async function GET( request: NextRequest, diff --git a/control-plane/src/app/api/search/route.ts b/control-plane/src/app/api/search/route.ts index a4222534..4fb844f9 100644 --- a/control-plane/src/app/api/search/route.ts +++ b/control-plane/src/app/api/search/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; -const DATAPLANE_URL = process.env.NEXT_PUBLIC_DATAPLANE_API_URL || 'http://localhost:8080'; +const DATAPLANE_URL = process.env.DATAPLANE_API_URL || 'http://localhost:8080'; export async function POST(request: NextRequest) { try { diff --git a/control-plane/src/app/api/stats/[agentId]/route.ts b/control-plane/src/app/api/stats/[agentId]/route.ts index 5192044a..bfb5aa74 100644 --- a/control-plane/src/app/api/stats/[agentId]/route.ts +++ b/control-plane/src/app/api/stats/[agentId]/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; -const DATAPLANE_URL = process.env.NEXT_PUBLIC_DATAPLANE_API_URL || 'http://localhost:8080'; +const DATAPLANE_URL = process.env.DATAPLANE_API_URL || 'http://localhost:8080'; export async function GET( request: NextRequest, diff --git a/control-plane/src/app/api/think/route.ts b/control-plane/src/app/api/think/route.ts index 1a18a1b7..98b0bbb1 100644 --- a/control-plane/src/app/api/think/route.ts +++ b/control-plane/src/app/api/think/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; -const DATAPLANE_URL = process.env.NEXT_PUBLIC_DATAPLANE_API_URL || 'http://localhost:8080'; +const DATAPLANE_URL = process.env.DATAPLANE_API_URL || 'http://localhost:8080'; export async function POST(request: NextRequest) { try { diff --git a/control-plane/src/lib/api.ts b/control-plane/src/lib/api.ts index 3b01a26c..862e5682 100644 --- a/control-plane/src/lib/api.ts +++ b/control-plane/src/lib/api.ts @@ -213,7 +213,7 @@ export class ServerDataplaneClient { private baseUrl: string; constructor() { - this.baseUrl = process.env.NEXT_PUBLIC_DATAPLANE_API_URL || 'http://localhost:8080'; + this.baseUrl = process.env.DATAPLANE_API_URL || 'http://localhost:8080'; } async fetchDataplane( diff --git a/docker/.gitignore b/local-db/.gitignore similarity index 100% rename from docker/.gitignore rename to local-db/.gitignore diff --git a/docker/docker-compose.yml b/local-db/docker-compose.yml similarity index 100% rename from docker/docker-compose.yml rename to local-db/docker-compose.yml diff --git a/docker/init-extensions.sql b/local-db/init-extensions.sql similarity index 100% rename from docker/init-extensions.sql rename to local-db/init-extensions.sql diff --git a/memora-dev/memora_dev/__init__.py b/memora-dev/memora_dev/__init__.py new file mode 100644 index 00000000..2693abda --- /dev/null +++ b/memora-dev/memora_dev/__init__.py @@ -0,0 +1 @@ +"""Utility scripts for Memora development.""" diff --git a/generate_openapi.py b/memora-dev/memora_dev/generate_openapi.py similarity index 94% rename from generate_openapi.py rename to memora-dev/memora_dev/generate_openapi.py index 7518a8be..fb2b88ee 100644 --- a/generate_openapi.py +++ b/memora-dev/memora_dev/generate_openapi.py @@ -9,9 +9,6 @@ import sys import os from pathlib import Path -# Add parent directory to path to import memory module -sys.path.insert(0, str(Path(__file__).parent)) - from memora.api import create_app from memora import TemporalSemanticMemory diff --git a/memora-dev/pyproject.toml b/memora-dev/pyproject.toml new file mode 100644 index 00000000..e6c5e84d --- /dev/null +++ b/memora-dev/pyproject.toml @@ -0,0 +1,18 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "memora-dev" +version = "0.1.0" +description = "Development utilities for Memora" +requires-python = ">=3.11" +dependencies = [ + "memora", +] + +[tool.hatch.build.targets.wheel] +packages = ["memora_dev"] + +[tool.uv.sources] +memora = { workspace = true } diff --git a/README.md b/memora/README.md similarity index 100% rename from README.md rename to memora/README.md diff --git a/alembic.ini b/memora/alembic.ini similarity index 100% rename from alembic.ini rename to memora/alembic.ini diff --git a/alembic/README b/memora/alembic/README similarity index 100% rename from alembic/README rename to memora/alembic/README diff --git a/alembic/env.py b/memora/alembic/env.py similarity index 92% rename from alembic/env.py rename to memora/alembic/env.py index 8c6641fc..167bd283 100644 --- a/alembic/env.py +++ b/memora/alembic/env.py @@ -23,14 +23,17 @@ def load_env(): if os.getenv("DATABASE_URL"): return + # Look for .env files in the parent directory (root of the workspace) + root_dir = Path(__file__).parent.parent.parent + # Default to local environment - env_file = ".env.local" - if Path(env_file).exists(): + env_file = root_dir / ".env.local" + if env_file.exists(): load_dotenv(env_file) else: # Fallback to dev - env_file = ".env.dev" - if Path(env_file).exists(): + env_file = root_dir / ".env.dev" + if env_file.exists(): load_dotenv(env_file) load_env() diff --git a/alembic/script.py.mako b/memora/alembic/script.py.mako similarity index 100% rename from alembic/script.py.mako rename to memora/alembic/script.py.mako diff --git a/alembic/versions/01f989db9079_fix_memory_links_entity_id_to_be_.py b/memora/alembic/versions/01f989db9079_fix_memory_links_entity_id_to_be_.py similarity index 100% rename from alembic/versions/01f989db9079_fix_memory_links_entity_id_to_be_.py rename to memora/alembic/versions/01f989db9079_fix_memory_links_entity_id_to_be_.py diff --git a/alembic/versions/0e96398aae9e_add_async_operations_table.py b/memora/alembic/versions/0e96398aae9e_add_async_operations_table.py similarity index 100% rename from alembic/versions/0e96398aae9e_add_async_operations_table.py rename to memora/alembic/versions/0e96398aae9e_add_async_operations_table.py diff --git a/alembic/versions/1a35a4fa1950_add_bm25_fulltext_search.py b/memora/alembic/versions/1a35a4fa1950_add_bm25_fulltext_search.py similarity index 100% rename from alembic/versions/1a35a4fa1950_add_bm25_fulltext_search.py rename to memora/alembic/versions/1a35a4fa1950_add_bm25_fulltext_search.py diff --git a/alembic/versions/2a76a5bc2f09_add_status_and_error_to_async_operations.py b/memora/alembic/versions/2a76a5bc2f09_add_status_and_error_to_async_operations.py similarity index 100% rename from alembic/versions/2a76a5bc2f09_add_status_and_error_to_async_operations.py rename to memora/alembic/versions/2a76a5bc2f09_add_status_and_error_to_async_operations.py diff --git a/alembic/versions/8c55f5602451_remove_entity_type_column.py b/memora/alembic/versions/8c55f5602451_remove_entity_type_column.py similarity index 100% rename from alembic/versions/8c55f5602451_remove_entity_type_column.py rename to memora/alembic/versions/8c55f5602451_remove_entity_type_column.py diff --git a/alembic/versions/af0413383b3e_initial_schema.py b/memora/alembic/versions/af0413383b3e_initial_schema.py similarity index 100% rename from alembic/versions/af0413383b3e_initial_schema.py rename to memora/alembic/versions/af0413383b3e_initial_schema.py diff --git a/memora/__init__.py b/memora/memora/__init__.py similarity index 100% rename from memora/__init__.py rename to memora/memora/__init__.py diff --git a/memora/api.py b/memora/memora/api.py similarity index 100% rename from memora/api.py rename to memora/memora/api.py diff --git a/memora/cross_encoder.py b/memora/memora/cross_encoder.py similarity index 100% rename from memora/cross_encoder.py rename to memora/memora/cross_encoder.py diff --git a/memora/embeddings.py b/memora/memora/embeddings.py similarity index 100% rename from memora/embeddings.py rename to memora/memora/embeddings.py diff --git a/memora/entity_resolver.py b/memora/memora/entity_resolver.py similarity index 100% rename from memora/entity_resolver.py rename to memora/memora/entity_resolver.py diff --git a/memora/fact_extraction.py b/memora/memora/fact_extraction.py similarity index 100% rename from memora/fact_extraction.py rename to memora/memora/fact_extraction.py diff --git a/memora/llm_wrapper.py b/memora/memora/llm_wrapper.py similarity index 100% rename from memora/llm_wrapper.py rename to memora/memora/llm_wrapper.py diff --git a/memora/models.py b/memora/memora/models.py similarity index 100% rename from memora/models.py rename to memora/memora/models.py diff --git a/memora/operations/__init__.py b/memora/memora/operations/__init__.py similarity index 100% rename from memora/operations/__init__.py rename to memora/memora/operations/__init__.py diff --git a/memora/operations/batch_operations.py b/memora/memora/operations/batch_operations.py similarity index 100% rename from memora/operations/batch_operations.py rename to memora/memora/operations/batch_operations.py diff --git a/memora/operations/embedding_operations.py b/memora/memora/operations/embedding_operations.py similarity index 100% rename from memora/operations/embedding_operations.py rename to memora/memora/operations/embedding_operations.py diff --git a/memora/operations/link_operations.py b/memora/memora/operations/link_operations.py similarity index 100% rename from memora/operations/link_operations.py rename to memora/memora/operations/link_operations.py diff --git a/memora/operations/think_operations.py b/memora/memora/operations/think_operations.py similarity index 100% rename from memora/operations/think_operations.py rename to memora/memora/operations/think_operations.py diff --git a/memora/remote_client.py b/memora/memora/remote_client.py similarity index 100% rename from memora/remote_client.py rename to memora/memora/remote_client.py diff --git a/memora/search/__init__.py b/memora/memora/search/__init__.py similarity index 100% rename from memora/search/__init__.py rename to memora/memora/search/__init__.py diff --git a/memora/search/reranking.py b/memora/memora/search/reranking.py similarity index 100% rename from memora/search/reranking.py rename to memora/memora/search/reranking.py diff --git a/memora/search/retrieval.py b/memora/memora/search/retrieval.py similarity index 100% rename from memora/search/retrieval.py rename to memora/memora/search/retrieval.py diff --git a/memora/search/temporal_extraction.py b/memora/memora/search/temporal_extraction.py similarity index 100% rename from memora/search/temporal_extraction.py rename to memora/memora/search/temporal_extraction.py diff --git a/memora/search_helpers.py b/memora/memora/search_helpers.py similarity index 100% rename from memora/search_helpers.py rename to memora/memora/search_helpers.py diff --git a/memora/search_trace.py b/memora/memora/search_trace.py similarity index 100% rename from memora/search_trace.py rename to memora/memora/search_trace.py diff --git a/memora/search_tracer.py b/memora/memora/search_tracer.py similarity index 100% rename from memora/search_tracer.py rename to memora/memora/search_tracer.py diff --git a/memora/task_backend.py b/memora/memora/task_backend.py similarity index 100% rename from memora/task_backend.py rename to memora/memora/task_backend.py diff --git a/memora/temporal_semantic_memory.py b/memora/memora/temporal_semantic_memory.py similarity index 100% rename from memora/temporal_semantic_memory.py rename to memora/memora/temporal_semantic_memory.py diff --git a/memora/utils.py b/memora/memora/utils.py similarity index 100% rename from memora/utils.py rename to memora/memora/utils.py diff --git a/memora/web/__init__.py b/memora/memora/web/__init__.py similarity index 100% rename from memora/web/__init__.py rename to memora/memora/web/__init__.py diff --git a/memora/web/server.py b/memora/memora/web/server.py similarity index 100% rename from memora/web/server.py rename to memora/memora/web/server.py diff --git a/memora/pyproject.toml b/memora/pyproject.toml new file mode 100644 index 00000000..5242f095 --- /dev/null +++ b/memora/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "memora" +version = "0.1.0" +description = "Temporal + Semantic + Entity Memory System for AI agents using PostgreSQL" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "asyncpg>=0.29.0", + "python-dotenv>=1.0.0", + "openai>=1.0.0", + "pydantic>=2.0.0", + "rich>=13.0.0", + "sentence-transformers>=2.2.0", + "pytest>=7.0.0", + "pytest-asyncio>=0.21.0", + "langchain-text-splitters>=0.3.0", + "fastapi[standard]>=0.120.3", + "uvicorn>=0.38.0", + "sqlalchemy>=2.0.44", + "alembic>=1.17.1", + "pgvector>=0.4.1", + "greenlet>=3.2.4", + "psycopg2-binary>=2.9.11", + "pytest-timeout>=2.4.0", + "dateparser>=1.2.0", + "tiktoken>=0.12.0", + "httpx>=0.27.0", +] + +[tool.hatch.build.targets.wheel] +packages = ["memora"] + +[tool.pytest.ini_options] +log_cli = true +log_cli_level = "INFO" +log_cli_format = "%(asctime)s %(levelname)s %(message)s" +log_cli_date_format = "%Y-%m-%d %H:%M:%S" +addopts = "--timeout 60 -p no:warnings" +asyncio_default_fixture_loop_scope = "function" diff --git a/tests/__init__.py b/memora/tests/__init__.py similarity index 100% rename from tests/__init__.py rename to memora/tests/__init__.py diff --git a/tests/conftest.py b/memora/tests/conftest.py similarity index 95% rename from tests/conftest.py rename to memora/tests/conftest.py index cc22b929..2971e0b0 100644 --- a/tests/conftest.py +++ b/memora/tests/conftest.py @@ -19,7 +19,8 @@ LOCAL_DB_URL = "postgresql://memora:memora_dev@localhost:5432/memora" # Load environment variables from .env.local at the start of test session def pytest_configure(config): """Load environment variables before running tests.""" - env_file = Path(__file__).parent.parent / ".env.local" + # Look for .env.local in the workspace root (two levels up from tests dir) + env_file = Path(__file__).parent.parent.parent / ".env.local" if env_file.exists(): load_dotenv(env_file) else: diff --git a/tests/fixtures/README.md b/memora/tests/fixtures/README.md similarity index 100% rename from tests/fixtures/README.md rename to memora/tests/fixtures/README.md diff --git a/tests/fixtures/locomo_conversation_sample.json b/memora/tests/fixtures/locomo_conversation_sample.json similarity index 100% rename from tests/fixtures/locomo_conversation_sample.json rename to memora/tests/fixtures/locomo_conversation_sample.json diff --git a/tests/test_batch_chunking.py b/memora/tests/test_batch_chunking.py similarity index 100% rename from tests/test_batch_chunking.py rename to memora/tests/test_batch_chunking.py diff --git a/tests/test_chunking.py b/memora/tests/test_chunking.py similarity index 100% rename from tests/test_chunking.py rename to memora/tests/test_chunking.py diff --git a/tests/test_document_tracking.py b/memora/tests/test_document_tracking.py similarity index 100% rename from tests/test_document_tracking.py rename to memora/tests/test_document_tracking.py diff --git a/tests/test_performance_tuning.py b/memora/tests/test_performance_tuning.py similarity index 100% rename from tests/test_performance_tuning.py rename to memora/tests/test_performance_tuning.py diff --git a/tests/test_search_trace.py b/memora/tests/test_search_trace.py similarity index 100% rename from tests/test_search_trace.py rename to memora/tests/test_search_trace.py diff --git a/tests/test_temporal_extraction.py b/memora/tests/test_temporal_extraction.py similarity index 100% rename from tests/test_temporal_extraction.py rename to memora/tests/test_temporal_extraction.py diff --git a/tests/test_think.py b/memora/tests/test_think.py similarity index 100% rename from tests/test_think.py rename to memora/tests/test_think.py diff --git a/openapi.json b/openapi.json index 137b3672..cb50e030 100644 --- a/openapi.json +++ b/openapi.json @@ -472,7 +472,7 @@ "Memory Storage" ], "summary": "Store multiple memories", - "description": "Store multiple memory items in batch with automatic fact extraction.\n\n Features:\n - Efficient batch processing\n - Automatic fact extraction from natural language\n - Entity recognition and linking\n - Document tracking with optional upsert\n - Temporal and semantic linking\n\n The system automatically:\n 1. Extracts semantic facts from the content\n 2. Generates embeddings\n 3. Deduplicates similar facts\n 4. Creates temporal, semantic, and entity links\n 5. Tracks document metadata", + "description": "Store multiple memory items in batch with automatic fact extraction.\n\n Features:\n - Efficient batch processing\n - Automatic fact extraction from natural language\n - Entity recognition and linking\n - Document tracking with automatic upsert (when document_id is provided)\n - Temporal and semantic linking\n\n The system automatically:\n 1. Extracts semantic facts from the content\n 2. Generates embeddings\n 3. Deduplicates similar facts\n 4. Creates temporal, semantic, and entity links\n 5. Tracks document metadata\n\n Note: If document_id is provided and already exists, the old document and its memory units will be deleted before creating new ones (upsert behavior).", "operationId": "api_batch_put_api_memories_batch_post", "requestBody": { "content": { @@ -514,7 +514,7 @@ "Memory Storage" ], "summary": "Store multiple memories asynchronously", - "description": "Store multiple memory items in batch asynchronously using the task backend.\n\n This endpoint returns immediately after queuing the task, without waiting for completion.\n The actual processing happens in the background.\n\n Features:\n - Immediate response (non-blocking)\n - Background processing via task queue\n - Efficient batch processing\n - Automatic fact extraction from natural language\n - Entity recognition and linking\n - Document tracking with optional upsert\n - Temporal and semantic linking\n\n The system automatically:\n 1. Queues the batch put task\n 2. Returns immediately with success=True, queued=True\n 3. Processes in background: extracts facts, generates embeddings, creates links", + "description": "Store multiple memory items in batch asynchronously using the task backend.\n\n This endpoint returns immediately after queuing the task, without waiting for completion.\n The actual processing happens in the background.\n\n Features:\n - Immediate response (non-blocking)\n - Background processing via task queue\n - Efficient batch processing\n - Automatic fact extraction from natural language\n - Entity recognition and linking\n - Document tracking with automatic upsert (when document_id is provided)\n - Temporal and semantic linking\n\n The system automatically:\n 1. Queues the batch put task\n 2. Returns immediately with success=True, queued=True\n 3. Processes in background: extracts facts, generates embeddings, creates links\n\n Note: If document_id is provided and already exists, the old document and its memory units will be deleted before creating new ones (upsert behavior).", "operationId": "api_batch_put_async_api_memories_batch_async_post", "requestBody": { "content": { @@ -776,23 +776,6 @@ } ], "title": "Document Id" - }, - "document_metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Document Metadata" - }, - "upsert": { - "type": "boolean", - "title": "Upsert", - "default": false } }, "type": "object", @@ -814,8 +797,7 @@ "content": "Bob went hiking yesterday", "event_date": "2024-01-15T10:00:00Z" } - ], - "upsert": false + ] } }, "BatchPutResponse": { @@ -1188,7 +1170,10 @@ "title": "Query" }, "fact_type": { - "type": "string", + "items": { + "type": "string" + }, + "type": "array", "title": "Fact Type" }, "agent_id": { @@ -1215,6 +1200,17 @@ "type": "boolean", "title": "Trace", "default": false + }, + "question_date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Question Date" } }, "type": "object", @@ -1226,9 +1222,13 @@ "description": "Request model for search endpoint.", "example": { "agent_id": "user123", - "fact_type": "world", + "fact_type": [ + "world", + "agent" + ], "max_tokens": 4096, "query": "What did Alice say about machine learning?", + "question_date": "2023-05-30T23:40:00", "reranker": "heuristic", "thinking_budget": 100, "trace": true diff --git a/pyproject.toml b/pyproject.toml index 9a787778..2bad4e60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,45 +1,5 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" +[tool.uv.workspace] +members = ["memora", "benchmarks", "memora-dev"] -[project] -name = "memora" -version = "0.1.0" -description = "Temporal + Semantic + Entity Memory System for AI agents using PostgreSQL" -readme = "README.md" -requires-python = ">=3.11" -dependencies = [ - "asyncpg>=0.29.0", - "python-dotenv>=1.0.0", - "openai>=1.0.0", - "pydantic>=2.0.0", - "rich>=13.0.0", - "sentence-transformers>=2.2.0", - "pytest>=7.0.0", - "pytest-asyncio>=0.21.0", - "langchain-text-splitters>=0.3.0", - "fastapi[standard]>=0.120.3", - "uvicorn>=0.38.0", - "sqlalchemy>=2.0.44", - "alembic>=1.17.1", - "pgvector>=0.4.1", - "greenlet>=3.2.4", - "psycopg2-binary>=2.9.11", - "pytest-timeout>=2.4.0", - "dateparser>=1.2.0", - "tiktoken>=0.12.0", - "httpx>=0.27.0", - "streamlit>=1.51.0", - "python-fasthtml>=0.12.33", -] - -[tool.hatch.build.targets.wheel] -packages = ["memora"] - -[tool.pytest.ini_options] -log_cli = true -log_cli_level = "INFO" -log_cli_format = "%(asctime)s %(levelname)s %(message)s" -log_cli_date_format = "%Y-%m-%d %H:%M:%S" -addopts = "--timeout 60 -p no:warnings" -asyncio_default_fixture_loop_scope = "function" +[tool.uv] +dev-dependencies = [] diff --git a/scripts/erase-local-db.sh b/scripts/erase-local-db.sh index a4a81330..30d0ce3e 100755 --- a/scripts/erase-local-db.sh +++ b/scripts/erase-local-db.sh @@ -7,7 +7,7 @@ echo "🛑 Stopping and erasing local PostgreSQL..." echo "" # Stop and remove containers, networks, volumes -cd docker +cd local-db docker-compose down -v echo "" diff --git a/scripts/migrate-db.sh b/scripts/migrate-db.sh index 0b5e5adb..f6680429 100755 --- a/scripts/migrate-db.sh +++ b/scripts/migrate-db.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../memora" echo "🔄 Database Migration Script" echo "============================" diff --git a/scripts/start-local-db.sh b/scripts/start-local-db.sh index 1fd9c8f2..dcd7f35f 100755 --- a/scripts/start-local-db.sh +++ b/scripts/start-local-db.sh @@ -7,7 +7,7 @@ echo "🚀 Starting local PostgreSQL..." echo "" # Start docker compose -cd docker +cd local-db docker-compose up -d echo "" @@ -25,7 +25,9 @@ cd .. export DATABASE_URL="postgresql://memora:memora_dev@localhost:5432/memora" echo "📊 Running database migrations..." +cd memora uv run alembic upgrade head +cd .. echo "" echo "✅ Database initialized successfully!" diff --git a/uv.lock b/uv.lock index f1485c60..2a01eadd 100644 --- a/uv.lock +++ b/uv.lock @@ -6,6 +6,13 @@ resolution-markers = [ "python_full_version < '3.12'", ] +[manifest] +members = [ + "benchmarks", + "memora", + "memora-dev", +] + [[package]] name = "alembic" version = "1.17.1" @@ -208,6 +215,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/fe/3aed5d0be4d404d12d36ab97e2f1791424d9ca39c2f754a6285d59a3b01d/beautifulsoup4-4.14.2-py3-none-any.whl", hash = "sha256:5ef6fa3a8cbece8488d66985560f97ed091e22bbc4e9c2338508a9d5de6d4515", size = 106392 }, ] +[[package]] +name = "benchmarks" +version = "0.1.0" +source = { editable = "benchmarks" } +dependencies = [ + { name = "memora" }, + { name = "openai" }, + { name = "pydantic" }, + { name = "python-fasthtml" }, + { name = "rich" }, + { name = "streamlit" }, +] + +[package.metadata] +requires-dist = [ + { name = "memora", editable = "memora" }, + { name = "openai", specifier = ">=1.0.0" }, + { name = "pydantic", specifier = ">=2.0.0" }, + { name = "python-fasthtml", specifier = ">=0.12.33" }, + { name = "rich", specifier = ">=13.0.0" }, + { name = "streamlit", specifier = ">=1.51.0" }, +] + [[package]] name = "blinker" version = "1.9.0" @@ -1015,7 +1045,7 @@ wheels = [ [[package]] name = "memora" version = "0.1.0" -source = { editable = "." } +source = { editable = "memora" } dependencies = [ { name = "alembic" }, { name = "asyncpg" }, @@ -1032,11 +1062,9 @@ dependencies = [ { name = "pytest-asyncio" }, { name = "pytest-timeout" }, { name = "python-dotenv" }, - { name = "python-fasthtml" }, { name = "rich" }, { name = "sentence-transformers" }, { name = "sqlalchemy" }, - { name = "streamlit" }, { name = "tiktoken" }, { name = "uvicorn" }, ] @@ -1058,15 +1086,24 @@ requires-dist = [ { name = "pytest-asyncio", specifier = ">=0.21.0" }, { name = "pytest-timeout", specifier = ">=2.4.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, - { name = "python-fasthtml", specifier = ">=0.12.33" }, { name = "rich", specifier = ">=13.0.0" }, { name = "sentence-transformers", specifier = ">=2.2.0" }, { name = "sqlalchemy", specifier = ">=2.0.44" }, - { name = "streamlit", specifier = ">=1.51.0" }, { name = "tiktoken", specifier = ">=0.12.0" }, { name = "uvicorn", specifier = ">=0.38.0" }, ] +[[package]] +name = "memora-dev" +version = "0.1.0" +source = { editable = "memora-dev" } +dependencies = [ + { name = "memora" }, +] + +[package.metadata] +requires-dist = [{ name = "memora", editable = "memora" }] + [[package]] name = "mpmath" version = "1.3.0"