diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ffd540da..3c3c99ac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,11 +6,8 @@ on: - 'v*' jobs: - build-python-packages: + build-python-package: runs-on: ubuntu-latest - strategy: - matrix: - package: [memora, benchmarks, memora-dev] steps: - uses: actions/checkout@v4 @@ -25,16 +22,16 @@ jobs: with: python-version-file: ".python-version" - - name: Build package + - name: Build memora package run: | - cd ${{ matrix.package }} + cd memora uv build - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: python-${{ matrix.package }}-dist - path: ${{ matrix.package }}/dist/* + name: python-memora-dist + path: memora/dist/* retention-days: 30 build-rust-cli: @@ -132,7 +129,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - component: [standalone, control-plane] + component: [api, control-plane] steps: - uses: actions/checkout@v4 @@ -144,24 +141,24 @@ jobs: id: get_version run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - - name: Build Docker image (standalone) - if: matrix.component == 'standalone' + - name: Build Docker image (api) + if: matrix.component == 'api' uses: docker/build-push-action@v6 with: context: . - file: standalone/Dockerfile + file: docker/api.Dockerfile push: false - tags: memora-standalone:${{ steps.get_version.outputs.VERSION }} + tags: memora-api:${{ steps.get_version.outputs.VERSION }} cache-from: type=gha cache-to: type=gha,mode=max - outputs: type=docker,dest=/tmp/memora-standalone.tar + outputs: type=docker,dest=/tmp/memora-api.tar - name: Build Docker image (control-plane) if: matrix.component == 'control-plane' uses: docker/build-push-action@v6 with: - context: ./memora-control-plane - file: memora-control-plane/Dockerfile + context: . + file: docker/control-plane.Dockerfile push: false tags: memora-control-plane:${{ steps.get_version.outputs.VERSION }} cache-from: type=gha @@ -201,25 +198,131 @@ jobs: path: helm-packages/*.tgz retention-days: 30 - create-release-summary: + create-github-release: runs-on: ubuntu-latest - needs: [build-python-packages, build-rust-cli, build-control-plane, build-docker-images, package-helm-chart] + needs: [build-python-package, build-rust-cli, build-control-plane, build-docker-images, package-helm-chart] + permissions: + contents: write steps: + - uses: actions/checkout@v4 + + - name: Extract version from tag + id: get_version + run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT + - name: Download all artifacts uses: actions/download-artifact@v4 with: path: ./artifacts + - name: Prepare release assets + run: | + mkdir -p release-assets + # Python package + cp artifacts/python-memora-dist/* release-assets/ + # Rust CLI binaries + cp artifacts/rust-cli-memora-linux-amd64/memora-linux-amd64 release-assets/ + cp artifacts/rust-cli-memora-darwin-amd64/memora-darwin-amd64 release-assets/ + cp artifacts/rust-cli-memora-darwin-arm64/memora-darwin-arm64 release-assets/ + # Helm chart + cp artifacts/helm-chart/*.tgz release-assets/ + # Docker images + cp artifacts/docker-image-api/memora-api.tar release-assets/ + cp artifacts/docker-image-control-plane/memora-control-plane.tar release-assets/ + # Control plane build + cd artifacts/control-plane-build + tar czf ../../release-assets/memora-control-plane-build.tar.gz . + cd ../.. + + - name: Generate release notes + id: release_notes + run: | + cat << EOF > release-notes.md + # Memora v${{ steps.get_version.outputs.VERSION }} + + ## 📦 Release Artifacts + + ### Python Package + - \`memora-${{ steps.get_version.outputs.VERSION }}-py3-none-any.whl\` + - \`memora-${{ steps.get_version.outputs.VERSION }}.tar.gz\` + + ### CLI Binaries + - \`memora-linux-amd64\` - Linux x86_64 + - \`memora-darwin-amd64\` - macOS Intel + - \`memora-darwin-arm64\` - macOS Apple Silicon + + ### Helm Chart + - \`memora-${{ steps.get_version.outputs.VERSION }}.tgz\` + + ### Docker Images + - \`memora-api.tar\` - API server Docker image + - \`memora-control-plane.tar\` - Control Plane Docker image + + ### Control Plane + - \`memora-control-plane-build.tar.gz\` - Next.js standalone build + + ## 🚀 Installation + + ### Python Package + \`\`\`bash + pip install memora==${{ steps.get_version.outputs.VERSION }} + \`\`\` + + ### CLI + \`\`\`bash + # macOS (Apple Silicon) + curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/memora-darwin-arm64 -o memora + chmod +x memora + sudo mv memora /usr/local/bin/ + + # macOS (Intel) + curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/memora-darwin-amd64 -o memora + chmod +x memora + sudo mv memora /usr/local/bin/ + + # Linux + curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/memora-linux-amd64 -o memora + chmod +x memora + sudo mv memora /usr/local/bin/ + \`\`\` + + ### Helm Chart + \`\`\`bash + helm install memora memora-${{ steps.get_version.outputs.VERSION }}.tgz + \`\`\` + + ### Docker + \`\`\`bash + # Load API image + docker load < memora-api.tar + + # Load Control Plane image + docker load < memora-control-plane.tar + \`\`\` + EOF + cat release-notes.md + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + files: release-assets/* + body_path: release-notes.md + draft: false + prerelease: false + generate_release_notes: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Create release summary run: | - echo "# Release Artifacts Built Successfully" >> $GITHUB_STEP_SUMMARY + echo "# Release v${{ steps.get_version.outputs.VERSION }} Published Successfully" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "## Components" >> $GITHUB_STEP_SUMMARY - echo "- ✅ Python packages (memora, benchmarks, memora-dev)" >> $GITHUB_STEP_SUMMARY + echo "## 📦 Components" >> $GITHUB_STEP_SUMMARY + echo "- ✅ Python package (memora)" >> $GITHUB_STEP_SUMMARY echo "- ✅ Rust CLI (Linux amd64, macOS amd64, macOS arm64)" >> $GITHUB_STEP_SUMMARY echo "- ✅ Control Plane Next.js application" >> $GITHUB_STEP_SUMMARY - echo "- ✅ Docker images (standalone, control-plane)" >> $GITHUB_STEP_SUMMARY + echo "- ✅ Docker images (API, Control Plane)" >> $GITHUB_STEP_SUMMARY echo "- ✅ Helm chart" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "All artifacts are available for download in the workflow artifacts." >> $GITHUB_STEP_SUMMARY + echo "🎉 Release is now available at: https://github.com/${{ github.repository }}/releases/tag/v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4e4ecc46..90036173 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,7 +29,7 @@ jobs: MEMORA_API_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/memora_test MEMORA_API_LLM_PROVIDER: groq MEMORA_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }} - MEMORA_API_LLM_MODEL: openai/gpt-oss-120b + MEMORA_API_LLM_MODEL: openai/gpt-oss-20b steps: - uses: actions/checkout@v4 diff --git a/memora-control-plane/.env.example b/memora-control-plane/.env.example deleted file mode 100644 index c706ab84..00000000 --- a/memora-control-plane/.env.example +++ /dev/null @@ -1,2 +0,0 @@ -# Dataplane API URL -MEMORA_CP_DATAPLANE_API_URL=http://localhost:8080 diff --git a/memora-control-plane/Dockerfile b/memora-control-plane/Dockerfile deleted file mode 100644 index 55309122..00000000 --- a/memora-control-plane/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -FROM node:20-alpine AS base - -# Install dependencies only when needed -FROM base AS deps -RUN apk add --no-cache libc6-compat -WORKDIR /app - -COPY package.json package-lock.json ./ -RUN npm ci - -# Rebuild the source code only when needed -FROM base AS builder -WORKDIR /app -COPY --from=deps /app/node_modules ./node_modules -COPY . . - -RUN npm run build - -# Production image, copy all the files and run next -FROM base AS runner -WORKDIR /app - -ENV NODE_ENV=production - -RUN addgroup --system --gid 1001 nodejs -RUN adduser --system --uid 1001 nextjs - -# Automatically leverage output traces to reduce image size -COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ -COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static - -USER nextjs - -EXPOSE 3000 - -ENV PORT=3000 -ENV HOSTNAME="0.0.0.0" - -CMD ["node", "server.js"] diff --git a/memora-control-plane/README.md b/memora-control-plane/README.md deleted file mode 100644 index 44a01d6c..00000000 --- a/memora-control-plane/README.md +++ /dev/null @@ -1,243 +0,0 @@ -# Memory Control Plane - -Control plane UI for the temporal semantic memory system built with Next.js, React, TypeScript, Tailwind CSS, and Cytoscape.js. - -## Overview - -The control plane is a modern web application that provides a comprehensive UI for managing and visualizing temporal semantic memories. It acts as a proxy between the browser and the Python FastAPI dataplane, eliminating CORS issues and providing a clean separation of concerns. - -## Architecture - -``` -Browser ←→ Control Plane (Next.js) ←→ Dataplane (Python FastAPI) -``` - -The control plane: -- Serves the React UI to the browser -- Provides Next.js API routes (`/api/*`) that proxy requests to the dataplane -- Handles client-side state management and visualization -- Eliminates CORS issues by serving both UI and API from the same origin - -## Features - -### 🔍 Search Debug (Most Important) -- **Multi-pane search interface**: Add multiple search panes for comparison -- **Interactive search controls**: Query, fact type, thinking budget, reranker selection, max tokens -- **Phase-based visualization**: Four phases of the retrieval pipeline - - **1. Retrieval**: View results from each method (Semantic, BM25, Graph, Temporal) with ranks and scores - - **2. RRF Merge**: See how Reciprocal Rank Fusion combines rankings from different methods - - **3. Reranking**: Compare before/after reranking with rank changes highlighted (blue = improved) - - **4. Final Results**: Detailed score breakdown with activation, similarity, recency, frequency ranks -- **Comprehensive stats**: Nodes visited, entry points, budget usage, results count, duration -- **Trace visualization**: See exactly how each retrieval method performs and contributes - -### 📊 Data Visualization -- **World Facts**: View and explore general knowledge memories -- **Agent Facts**: Track agent actions and activities -- **Opinions**: Monitor agent beliefs and perspectives -- **Documents**: Manage source documents - -Each fact type supports: -- Interactive graph visualization with Cytoscape.js (circle, grid, force-directed layouts) -- Searchable table view with filtering -- Real-time data loading - -### 💭 Think Interface -- Ask questions to the AI agent -- View source facts used (world, agent, opinions) -- See newly formed opinions with confidence scores -- Configurable thinking budget - -### ➕ Add Memory -- Submit new memories with context -- Support for event dates and document metadata -- Sync or async processing options -- Upsert capability for updates - -### 📈 Statistics & Operations -- Real-time memory statistics (nodes, links, documents) -- Breakdown by fact type and link type -- Async operation monitoring (pending/failed) -- Auto-refresh every 5 seconds - -## Getting Started - -### Prerequisites - -- Node.js 18.x or later -- A running dataplane API server (Python FastAPI) - -### Installation - -```bash -npm install -``` - -### Configuration - -Configure the dataplane URL in `.env` (optional, defaults to http://localhost:8080): - -```bash -cat > .env << 'EOF' -MEMORA_CP_DATAPLANE_API_URL=http://localhost:8080 -EOF -``` - -### Development - -**Terminal 1 - Start Dataplane:** -```bash -# From project root -./scripts/start-server.sh -``` - -**Terminal 2 - Start Control Plane:** -```bash -cd control-plane -npm run dev -``` - -Open [http://localhost:3000](http://localhost:3000) in your browser. - -### Production Build - -```bash -npm run build -npm start -``` - -## Tech Stack - -- **Framework**: Next.js 16 with App Router -- **Language**: TypeScript -- **Styling**: Tailwind CSS v4 -- **Visualization**: Cytoscape.js -- **State Management**: React Context API -- **API**: Next.js API Routes (proxy to dataplane) - -## Project Structure - -``` -control-plane/ -├── src/ -│ ├── app/ -│ │ ├── api/ # API routes (proxy to dataplane) -│ │ │ ├── agents/ # GET /api/agents -│ │ │ ├── graph/ # GET /api/graph -│ │ │ ├── list/ # GET /api/list -│ │ │ ├── search/ # POST /api/search -│ │ │ ├── think/ # POST /api/think -│ │ │ ├── memories/ -│ │ │ │ ├── batch/ # POST /api/memories/batch -│ │ │ │ └── batch_async/ # POST /api/memories/batch_async -│ │ │ ├── documents/ -│ │ │ │ ├── route.ts # GET /api/documents -│ │ │ │ └── [documentId]/ # GET /api/documents/:id -│ │ │ ├── stats/ -│ │ │ │ └── [agentId]/ # GET /api/stats/:id -│ │ │ └── operations/ -│ │ │ └── [agentId]/ # GET /api/operations/:id -│ │ ├── dashboard/ -│ │ │ └── page.tsx # Main dashboard -│ │ ├── layout.tsx # Root layout -│ │ ├── page.tsx # Home (redirects to dashboard) -│ │ └── globals.css # Global styles -│ ├── components/ -│ │ ├── agent-selector.tsx # Agent dropdown -│ │ ├── data-view.tsx # Graph/table visualization -│ │ ├── documents-view.tsx # Document management -│ │ ├── think-view.tsx # AI thinking interface -│ │ ├── add-memory-view.tsx # Memory submission form -│ │ └── stats-view.tsx # Statistics dashboard -│ └── lib/ -│ ├── agent-context.tsx # Global agent state -│ ├── api.ts # API client -│ └── utils.ts # Utilities -├── .env # Environment config (optional) -└── package.json -``` - -## API Routes - -All control plane API routes proxy to the dataplane: - -| Route | Method | Description | -|-------|--------|-------------| -| `/api/agents` | GET | List all agents | -| `/api/graph` | GET | Get graph data for visualization | -| `/api/list` | GET | List memory units with search | -| `/api/search` | POST | Search memories | -| `/api/think` | POST | Generate AI answers | -| `/api/memories/batch` | POST | Store memories (sync) | -| `/api/memories/batch_async` | POST | Store memories (async) | -| `/api/documents` | GET | List documents | -| `/api/documents/:id` | GET | Get document details | -| `/api/stats/:agentId` | GET | Get agent statistics | -| `/api/operations/:agentId` | GET | List async operations | - -## Usage - -### Using Search Debug (Primary Feature) -1. Go to the **Search Debug** tab -2. Enter a search query -3. Select fact type (World, Agent, Opinion) -4. Adjust thinking budget, reranker (Heuristic/Cross-Encoder), and max tokens -5. Click **Search** to run the query -6. Use the phase radio buttons to explore the retrieval pipeline: - - **1. Retrieval**: Switch between Semantic/BM25/Graph/Temporal tabs to see each method's results - - **2. RRF Merge**: View how rankings from different methods are combined with source ranks - - **3. Reranking**: See rank changes (↑ improved, ↓ declined) with score component breakdowns - - **4. Final Results**: Detailed table with all score components and individual metric ranks -7. Monitor the status bar showing nodes visited, entry points, budget usage, and duration -8. Add more panes with **+ Add Search Pane** to compare different queries side-by-side -9. Each pane maintains independent state for query, settings, and current phase view - -### Selecting an Agent -1. Use the dropdown in the top navigation bar -2. Select an agent to view their memories -3. All views will automatically filter by the selected agent - -### Visualizing Memories -1. Go to the **Data** tab -2. Choose a fact type (World, Agent, Opinions, or Documents) -3. Click **Load** to fetch data -4. Toggle between **Graph** and **Table** views -5. Use search to filter results - -### Asking Questions -1. Go to the **Think** tab -2. Enter your question -3. Adjust thinking budget if needed -4. Click **Think** to get an AI-generated answer -5. View source facts and new opinions formed - -### Adding Memories -1. Go to the **Add Memory** tab -2. Enter memory content (required) -3. Optionally add context, date, document metadata -4. Choose sync or async processing -5. Click **Submit Memory** - -### Monitoring Stats -1. Go to the **Stats & Operations** tab -2. View real-time statistics -3. Monitor pending/failed async operations -4. Stats auto-refresh every 5 seconds - -## Development Notes - -- The control plane uses client-side rendering for interactive features -- API routes run on the server and proxy to the dataplane -- No direct browser-to-dataplane communication (no CORS issues) -- Graph visualization uses Cytoscape.js with multiple layout options -- Tailwind CSS v4 for styling (simplified configuration) - -## Troubleshooting - -**CORS Errors**: The control plane should eliminate CORS issues. If you see them, ensure you're accessing the control plane at `http://localhost:3000` (not the dataplane directly). - -**Connection Errors**: Verify the dataplane is running at the URL specified in `.env` (default: `http://localhost:8080`). - -**Graph Not Rendering**: Check browser console for errors. Ensure data is loading correctly from `/api/graph`. - -**Build Warnings**: The "workspace root" warning about lockfiles is harmless and can be ignored. diff --git a/memora-control-plane/build-docker.sh b/memora-control-plane/build-docker.sh deleted file mode 100755 index 770c5c9a..00000000 --- a/memora-control-plane/build-docker.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -set -e - -# Default values -IMAGE_NAME="memora-control-plane" -IMAGE_TAG="latest" -REGISTRY="" - -# Parse command line arguments -while [[ $# -gt 0 ]]; do - case $1 in - --name) - IMAGE_NAME="$2" - shift 2 - ;; - --tag) - IMAGE_TAG="$2" - shift 2 - ;; - --registry) - REGISTRY="$2" - shift 2 - ;; - --help) - echo "Usage: $0 [OPTIONS]" - echo "" - echo "Options:" - echo " --name NAME Docker image name (default: control-plane)" - echo " --tag TAG Docker image tag (default: latest)" - echo " --registry REG Docker registry URL (optional)" - echo " --help Show this help message" - echo "" - echo "Example:" - echo " $0 --name myapp --tag v1.0.0" - echo " $0 --registry docker.io/myuser --name control-plane --tag v1.0.0" - exit 0 - ;; - *) - echo "Unknown option: $1" - echo "Use --help for usage information" - exit 1 - ;; - esac -done - -# Construct full image name -if [ -n "$REGISTRY" ]; then - FULL_IMAGE_NAME="${REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}" -else - FULL_IMAGE_NAME="${IMAGE_NAME}:${IMAGE_TAG}" -fi - -echo "Building Docker image: ${FULL_IMAGE_NAME}" -echo "========================================" - -# Build the Docker image -docker build -t "${FULL_IMAGE_NAME}" . - -echo "" -echo "Build completed successfully!" -echo "Image: ${FULL_IMAGE_NAME}" -echo "" -echo "To run the container:" -echo " docker run -p 3000:3000 \\" -echo " -e MEMORA_CP_DATAPLANE_API_URL=http://your-api-url:8080 \\" -echo " ${FULL_IMAGE_NAME}" -echo "" -echo "Or with an env file:" -echo " docker run -p 3000:3000 --env-file .env ${FULL_IMAGE_NAME}" -echo "" -echo "To push to registry (if registry specified):" -if [ -n "$REGISTRY" ]; then - echo " docker push ${FULL_IMAGE_NAME}" -fi diff --git a/memora-control-plane/start-server.sh b/memora-control-plane/start-server.sh deleted file mode 100755 index f1c4560e..00000000 --- a/memora-control-plane/start-server.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -# Wrapper script to map MEMORA_CP_* environment variables to Next.js standard variables - -# Map prefixed env vars to standard Next.js env vars -export HOSTNAME="${MEMORA_CP_HOSTNAME:-0.0.0.0}" -export PORT="${MEMORA_CP_PORT:-3000}" - -# Start the Next.js server -# The server.js is in the standalone output at the root -exec node server.js diff --git a/memora-dev/README.md b/memora-dev/README.md new file mode 100644 index 00000000..db846939 --- /dev/null +++ b/memora-dev/README.md @@ -0,0 +1 @@ +# Memory Dev/Utils scripts \ No newline at end of file diff --git a/memora/README.md b/memora/README.md new file mode 100644 index 00000000..2fd67659 --- /dev/null +++ b/memora/README.md @@ -0,0 +1 @@ +# Memory \ No newline at end of file diff --git a/scripts/dev/start-control-plane.sh b/scripts/dev/start-control-plane.sh index 2344a2c6..7910ea4d 100755 --- a/scripts/dev/start-control-plane.sh +++ b/scripts/dev/start-control-plane.sh @@ -1,40 +1,12 @@ #!/bin/bash set -e -cd "$(dirname "$0")/../../memora-control-plane" - -# Parse arguments -PORT=3000 - -while [[ $# -gt 0 ]]; do - case $1 in - --port|-p) - PORT="$2" - shift 2 - ;; - --help|-h) - echo "Usage: $0 [options]" - echo "" - echo "Options:" - echo " --port, -p PORT Port to run on (default: 3000)" - echo " --help, -h Show this help message" - echo "" - echo "Example:" - echo " $0 --port 3001" - exit 0 - ;; - *) - echo "Unknown option: $1" - echo "Use --help for usage information" - exit 1 - ;; - esac -done +ROOT_DIR="$(git rev-parse --show-toplevel)" +cd "$ROOT_DIR/memora-control-plane" || exit 1 # Check if .env exists in workspace root -ROOT_DIR="$(dirname "$0")/../.." if [ ! -f "$ROOT_DIR/.env" ]; then - echo "⚠️ Warning: .env not found in workspace root" + echo "⚠️ Warning: .env not found in workspace root at $ROOT_DIR/.env" echo "📝 Please create a .env file if you need to set MEMORA_CP_DATAPLANE_API_URL" echo " Default will use http://localhost:8080" echo "" @@ -48,6 +20,7 @@ if [ -f "$ROOT_DIR/.env" ]; then source "$ROOT_DIR/.env" set +a fi +PORT=3000 echo "" echo "Control plane will be available at: http://localhost:${PORT}" echo "" diff --git a/scripts/release.sh b/scripts/release.sh index e3bce381..9c203648 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -65,7 +65,7 @@ fi print_info "Updating version in all components..." # Update Python packages -PYTHON_PACKAGES=("memora" "benchmarks" "memora-dev") +PYTHON_PACKAGES=("memora" "memora-dev" "memora-dev/benchmarks") for package in "${PYTHON_PACKAGES[@]}"; do PYPROJECT_FILE="$package/pyproject.toml" if [ -f "$PYPROJECT_FILE" ]; then @@ -128,7 +128,7 @@ git add -A git commit -m "Release v$VERSION - Update version to $VERSION in all components -- Python packages: memora, benchmarks, memora-dev +- Python packages: memora, memora-dev, memora-dev/benchmarks - Rust CLI: memora-cli - Control Plane: memora-control-plane - Helm chart"