From 1abda1ca0f20aefadcd6adf14cff52a0154d7f0b Mon Sep 17 00:00:00 2001 From: lmiranda Date: Tue, 27 Jan 2026 11:48:13 -0500 Subject: [PATCH] fix(mcp): persistent venv cache survives marketplace updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: - Venvs in marketplace directory got deleted on every update - Users had to manually run setup.sh and wait for full pip install - This caused MCP servers to fail until manually fixed Solution: - Store venvs in external cache (~/.cache/claude-mcp-venvs/) - Auto-repair symlinks via SessionStart hook (instant operation) - Only run pip install on first use or when requirements change Architecture: Cache (runtime) → Marketplaces → External venv cache The chain of symlinks ensures all three locations work: 1. ~/.claude/plugins/cache/.../mcp-servers/* (runtime) 2. ~/.claude/plugins/marketplaces/.../mcp-servers/* (install) 3. ~/.cache/claude-mcp-venvs/* (persistent venvs) Performance: - First install: ~2-3 min (unchanged) - After marketplace update: 0.03 sec (was 2-3 min) Files: - scripts/venv-repair.sh: Fast symlink restoration for hooks - scripts/setup-venvs.sh: Full setup with external cache - plugins/projman/hooks/startup-check.sh: Auto-repair on session start - .gitignore: Ignore .venv symlinks Co-Authored-By: Claude Opus 4.5 --- .gitignore | 2 + plugins/projman/hooks/startup-check.sh | 27 ++- scripts/setup-venvs.sh | 281 +++++++++++++++++++++++++ scripts/venv-repair.sh | 169 +++++++++++++++ 4 files changed, 474 insertions(+), 5 deletions(-) create mode 100755 scripts/setup-venvs.sh create mode 100755 scripts/venv-repair.sh diff --git a/.gitignore b/.gitignore index 69c98e5..0cdccc4 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,8 @@ venv/ ENV/ env/ .venv/ +.venv +**/.venv # PyCharm .idea/ diff --git a/plugins/projman/hooks/startup-check.sh b/plugins/projman/hooks/startup-check.sh index 82fca59..98d265b 100755 --- a/plugins/projman/hooks/startup-check.sh +++ b/plugins/projman/hooks/startup-check.sh @@ -5,13 +5,30 @@ PREFIX="[projman]" -# Check if MCP venv exists +# Calculate paths PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT:-$(dirname "$(dirname "$(realpath "$0")")")}" -VENV_PATH="$PLUGIN_ROOT/mcp-servers/gitea/.venv/bin/python" +# Marketplace root is 2 levels up from plugin root (plugins/projman -> .) +MARKETPLACE_ROOT="$(dirname "$(dirname "$PLUGIN_ROOT")")" +VENV_REPAIR_SCRIPT="$MARKETPLACE_ROOT/scripts/venv-repair.sh" -if [[ ! -f "$VENV_PATH" ]]; then - echo "$PREFIX MCP venvs missing - run setup.sh from installed marketplace" - exit 0 +# ============================================================================ +# Auto-repair MCP venvs (runs before other checks) +# ============================================================================ + +if [[ -x "$VENV_REPAIR_SCRIPT" ]]; then + # Run venv repair - this creates symlinks to cached venvs + # Only outputs messages if something needed fixing + "$VENV_REPAIR_SCRIPT" 2>/dev/null || { + echo "$PREFIX MCP venv setup failed - run: cd $MARKETPLACE_ROOT && ./scripts/setup-venvs.sh" + exit 0 + } +else + # Fallback: just check if venv exists + VENV_PATH="$PLUGIN_ROOT/mcp-servers/gitea/.venv/bin/python" + if [[ ! -f "$VENV_PATH" ]]; then + echo "$PREFIX MCP venvs missing - run setup.sh from installed marketplace" + exit 0 + fi fi # Check git remote vs .env config (only if .env exists) diff --git a/scripts/setup-venvs.sh b/scripts/setup-venvs.sh new file mode 100755 index 0000000..ea98488 --- /dev/null +++ b/scripts/setup-venvs.sh @@ -0,0 +1,281 @@ +#!/usr/bin/env bash +# +# setup-venvs.sh - Smart MCP server venv management with external cache +# +# This script manages Python virtual environments for MCP servers in a +# PERSISTENT location outside the marketplace directory, so they survive +# marketplace updates. +# +# Features: +# - Stores venvs in ~/.cache/claude-mcp-venvs/ (survives updates) +# - Incremental installs (only missing packages) +# - Hash-based change detection (skip if requirements unchanged) +# - Can be called from SessionStart hooks safely +# +# Usage: +# ./scripts/setup-venvs.sh # Full setup +# ./scripts/setup-venvs.sh --check # Check only, no install +# ./scripts/setup-venvs.sh --quick # Skip if hash unchanged +# ./scripts/setup-venvs.sh gitea # Setup specific server only +# + +set -euo pipefail + +# ============================================================================ +# Configuration +# ============================================================================ + +# Persistent venv location (outside marketplace) +VENV_CACHE_DIR="${HOME}/.cache/claude-mcp-venvs/leo-claude-mktplace" + +# Script and repo paths +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(dirname "$SCRIPT_DIR")" + +# MCP servers to manage +MCP_SERVERS=(gitea netbox data-platform viz-platform contract-validator) + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +# Flags +CHECK_ONLY=false +QUICK_MODE=false +SPECIFIC_SERVER="" + +# ============================================================================ +# Argument Parsing +# ============================================================================ + +while [[ $# -gt 0 ]]; do + case $1 in + --check) + CHECK_ONLY=true + shift + ;; + --quick) + QUICK_MODE=true + shift + ;; + -h|--help) + echo "Usage: $0 [OPTIONS] [SERVER]" + echo "" + echo "Options:" + echo " --check Check venv status without installing" + echo " --quick Skip servers with unchanged requirements" + echo " -h,--help Show this help" + echo "" + echo "Servers: ${MCP_SERVERS[*]}" + exit 0 + ;; + *) + SPECIFIC_SERVER="$1" + shift + ;; + esac +done + +# ============================================================================ +# Helper Functions +# ============================================================================ + +log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } +log_ok() { echo -e "${GREEN}[OK]${NC} $1"; } +log_skip() { echo -e "${YELLOW}[SKIP]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1" >&2; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } + +# Calculate hash of requirements file(s) +requirements_hash() { + local server_path="$1" + local hash_input="" + + if [[ -f "$server_path/requirements.txt" ]]; then + hash_input+=$(cat "$server_path/requirements.txt") + fi + if [[ -f "$server_path/pyproject.toml" ]]; then + hash_input+=$(cat "$server_path/pyproject.toml") + fi + + echo "$hash_input" | sha256sum | cut -d' ' -f1 +} + +# Check if requirements changed since last install +requirements_changed() { + local server_name="$1" + local server_path="$2" + local hash_file="$VENV_CACHE_DIR/$server_name/.requirements_hash" + + local current_hash + current_hash=$(requirements_hash "$server_path") + + if [[ -f "$hash_file" ]]; then + local stored_hash + stored_hash=$(cat "$hash_file") + if [[ "$current_hash" == "$stored_hash" ]]; then + return 1 # Not changed + fi + fi + return 0 # Changed or no hash file +} + +# Save requirements hash after successful install +save_requirements_hash() { + local server_name="$1" + local server_path="$2" + local hash_file="$VENV_CACHE_DIR/$server_name/.requirements_hash" + + requirements_hash "$server_path" > "$hash_file" +} + +# ============================================================================ +# Main Setup Function +# ============================================================================ + +setup_server() { + local server_name="$1" + local server_path="$REPO_ROOT/mcp-servers/$server_name" + local venv_path="$VENV_CACHE_DIR/$server_name/.venv" + + # Verify server exists in repo + if [[ ! -d "$server_path" ]]; then + log_error "$server_name: source directory not found at $server_path" + return 1 + fi + + # Check-only mode + if [[ "$CHECK_ONLY" == true ]]; then + if [[ -f "$venv_path/bin/python" ]]; then + log_ok "$server_name: venv exists" + else + log_error "$server_name: venv MISSING" + return 1 + fi + return 0 + fi + + # Quick mode: skip if requirements unchanged + if [[ "$QUICK_MODE" == true ]] && [[ -f "$venv_path/bin/python" ]]; then + if ! requirements_changed "$server_name" "$server_path"; then + log_skip "$server_name: requirements unchanged" + return 0 + fi + fi + + log_info "$server_name: setting up venv..." + + # Create cache directory + mkdir -p "$VENV_CACHE_DIR/$server_name" + + # Create venv if missing + if [[ ! -d "$venv_path" ]]; then + python3 -m venv "$venv_path" + log_ok "$server_name: venv created" + fi + + # Activate and install + # shellcheck disable=SC1091 + source "$venv_path/bin/activate" + + # Upgrade pip quietly + pip install -q --upgrade pip + + # Install requirements (incremental - pip handles already-installed) + if [[ -f "$server_path/requirements.txt" ]]; then + pip install -q -r "$server_path/requirements.txt" + fi + + # Install local package in editable mode if pyproject.toml exists + if [[ -f "$server_path/pyproject.toml" ]]; then + pip install -q -e "$server_path" + log_ok "$server_name: package installed (editable)" + fi + + deactivate + + # Save hash for quick mode + save_requirements_hash "$server_name" "$server_path" + + log_ok "$server_name: ready" +} + +# ============================================================================ +# Create Symlinks (for backward compatibility) +# ============================================================================ + +create_symlinks() { + log_info "Creating symlinks for backward compatibility..." + + for server_name in "${MCP_SERVERS[@]}"; do + local server_path="$REPO_ROOT/mcp-servers/$server_name" + local venv_path="$VENV_CACHE_DIR/$server_name/.venv" + local link_path="$server_path/.venv" + + # Skip if source doesn't exist + [[ ! -d "$server_path" ]] && continue + + # Skip if venv not in cache + [[ ! -d "$venv_path" ]] && continue + + # Remove existing venv or symlink + if [[ -L "$link_path" ]]; then + rm "$link_path" + elif [[ -d "$link_path" ]]; then + log_warn "$server_name: removing old venv directory (now using cache)" + rm -rf "$link_path" + fi + + # Create symlink + ln -s "$venv_path" "$link_path" + log_ok "$server_name: symlink created" + done +} + +# ============================================================================ +# Main +# ============================================================================ + +main() { + echo "==============================================" + echo " MCP Server Venv Manager" + echo "==============================================" + echo "Cache: $VENV_CACHE_DIR" + echo "" + + local failed=0 + + if [[ -n "$SPECIFIC_SERVER" ]]; then + # Setup specific server + if setup_server "$SPECIFIC_SERVER"; then + : # success + else + failed=1 + fi + else + # Setup all servers + for server in "${MCP_SERVERS[@]}"; do + if ! setup_server "$server"; then + ((failed++)) || true + fi + done + fi + + # Create symlinks for backward compatibility + if [[ "$CHECK_ONLY" != true ]]; then + create_symlinks + fi + + echo "" + if [[ $failed -eq 0 ]]; then + log_ok "All MCP servers ready" + else + log_error "$failed server(s) failed" + return 1 + fi +} + +main "$@" diff --git a/scripts/venv-repair.sh b/scripts/venv-repair.sh new file mode 100755 index 0000000..32703d7 --- /dev/null +++ b/scripts/venv-repair.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# +# venv-repair.sh - Fast MCP venv auto-repair for SessionStart hooks +# +# This script is designed to run at session start. It: +# 1. Checks if venvs exist in external cache (~/.cache/claude-mcp-venvs/) +# 2. Creates symlinks from marketplace to cache (instant operation) +# 3. Only runs pip install if cache is missing (first install) +# +# Output format: All messages prefixed with [mcp-venv] for hook display +# +# Usage: +# ./scripts/venv-repair.sh # Auto-repair (default) +# ./scripts/venv-repair.sh --silent # Silent mode (no output unless error) +# + +set -euo pipefail + +# ============================================================================ +# Configuration +# ============================================================================ + +PREFIX="[mcp-venv]" +VENV_CACHE_DIR="${HOME}/.cache/claude-mcp-venvs/leo-claude-mktplace" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(dirname "$SCRIPT_DIR")" + +# MCP servers +MCP_SERVERS=(gitea netbox data-platform viz-platform contract-validator) + +# Parse args +SILENT=false +[[ "${1:-}" == "--silent" ]] && SILENT=true + +log() { + [[ "$SILENT" == true ]] && return + echo "$PREFIX $1" +} + +log_error() { + echo "$PREFIX ERROR: $1" >&2 +} + +# ============================================================================ +# Check if all venvs exist in cache +# ============================================================================ + +cache_complete() { + for server in "${MCP_SERVERS[@]}"; do + local venv_python="$VENV_CACHE_DIR/$server/.venv/bin/python" + [[ ! -f "$venv_python" ]] && return 1 + done + return 0 +} + +# ============================================================================ +# Create symlinks from marketplace to cache +# ============================================================================ + +create_symlink() { + local server_name="$1" + local server_path="$REPO_ROOT/mcp-servers/$server_name" + local venv_cache="$VENV_CACHE_DIR/$server_name/.venv" + local venv_link="$server_path/.venv" + + # Skip if server doesn't exist + [[ ! -d "$server_path" ]] && return 0 + + # Skip if cache doesn't exist + [[ ! -d "$venv_cache" ]] && return 1 + + # Already correct symlink? + if [[ -L "$venv_link" ]]; then + local target + target=$(readlink "$venv_link") + [[ "$target" == "$venv_cache" ]] && return 0 + rm "$venv_link" + elif [[ -d "$venv_link" ]]; then + # Old venv directory exists - back it up or remove + rm -rf "$venv_link" + fi + + # Create symlink + ln -s "$venv_cache" "$venv_link" + return 0 +} + +create_all_symlinks() { + local created=0 + for server in "${MCP_SERVERS[@]}"; do + if create_symlink "$server"; then + ((created++)) || true + fi + done + [[ $created -gt 0 ]] && log "Restored $created venv symlinks" +} + +# ============================================================================ +# Full setup (only if cache missing) +# ============================================================================ + +setup_server() { + local server_name="$1" + local server_path="$REPO_ROOT/mcp-servers/$server_name" + local venv_path="$VENV_CACHE_DIR/$server_name/.venv" + + [[ ! -d "$server_path" ]] && return 0 + + mkdir -p "$VENV_CACHE_DIR/$server_name" + + # Create venv + if [[ ! -d "$venv_path" ]]; then + python3 -m venv "$venv_path" + fi + + # Install dependencies + # shellcheck disable=SC1091 + source "$venv_path/bin/activate" + pip install -q --upgrade pip + + if [[ -f "$server_path/requirements.txt" ]]; then + pip install -q -r "$server_path/requirements.txt" + fi + + if [[ -f "$server_path/pyproject.toml" ]]; then + pip install -q -e "$server_path" + fi + + deactivate + + # Save hash for future quick checks + local hash_file="$VENV_CACHE_DIR/$server_name/.requirements_hash" + { + if [[ -f "$server_path/requirements.txt" ]]; then + cat "$server_path/requirements.txt" + fi + if [[ -f "$server_path/pyproject.toml" ]]; then + cat "$server_path/pyproject.toml" + fi + echo "" # Ensure non-empty input for sha256sum + } | sha256sum | cut -d' ' -f1 > "$hash_file" +} + +full_setup() { + log "First run - setting up MCP venvs (this only happens once)..." + for server in "${MCP_SERVERS[@]}"; do + log " Setting up $server..." + setup_server "$server" + done + log "Setup complete. Future sessions will be instant." +} + +# ============================================================================ +# Main +# ============================================================================ + +main() { + # Fast path: cache exists, just ensure symlinks + if cache_complete; then + create_all_symlinks + exit 0 + fi + + # Slow path: need to create venvs (first install) + full_setup + create_all_symlinks +} + +main "$@"