feat: Add caching layer and batch operations for improved performance
Implement Phase 3 improvements: intelligent caching and batch operations
to significantly enhance SDK performance and usability.
**1. Caching Layer Implementation**
Added complete caching infrastructure with LRU eviction and TTL support:
- `wikijs/cache/base.py`: Abstract BaseCache interface with CacheKey structure
- `wikijs/cache/memory.py`: MemoryCache implementation with:
* LRU (Least Recently Used) eviction policy
* Configurable TTL (time-to-live) expiration
* Cache statistics (hits, misses, hit rate)
* Resource-specific invalidation
* Automatic cleanup of expired entries
**Cache Integration:**
- Modified `WikiJSClient` to accept optional `cache` parameter
- Integrated caching into `PagesEndpoint.get()`:
* Check cache before API request
* Store successful responses in cache
* Invalidate cache on write operations (update, delete)
**2. Batch Operations**
Added efficient batch methods to Pages API:
- `create_many(pages_data)`: Batch create multiple pages
- `update_many(updates)`: Batch update pages with partial success handling
- `delete_many(page_ids)`: Batch delete with detailed error reporting
All batch methods include:
- Partial success support (continue on errors)
- Detailed error tracking with indices
- Comprehensive error messages
**3. Comprehensive Testing**
Added 27 new tests (all passing):
- `tests/test_cache.py`: 17 tests for caching (99% coverage)
* CacheKey string generation
* TTL expiration
* LRU eviction policy
* Cache invalidation (specific & all resources)
* Statistics tracking
- `tests/endpoints/test_pages_batch.py`: 10 tests for batch operations
* Successful batch creates/updates/deletes
* Partial failure handling
* Empty list edge cases
* Validation error handling
**Performance Benefits:**
- Caching reduces API calls for frequently accessed pages
- Batch operations reduce network overhead for bulk actions
- Configurable cache size and TTL for optimization
**Example Usage:**
```python
from wikijs import WikiJSClient
from wikijs.cache import MemoryCache
# Enable caching
cache = MemoryCache(ttl=300, max_size=1000)
client = WikiJSClient('https://wiki.example.com', auth='key', cache=cache)
# Cached GET requests
page = client.pages.get(123) # Fetches from API
page = client.pages.get(123) # Returns from cache
# Batch operations
pages = client.pages.create_many([
PageCreate(title="Page 1", path="page-1", content="Content 1"),
PageCreate(title="Page 2", path="page-2", content="Content 2"),
])
updates = client.pages.update_many([
{"id": 1, "content": "Updated content"},
{"id": 2, "is_published": False},
])
result = client.pages.delete_many([1, 2, 3])
print(f"Deleted {result['successful']} pages")
```
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
299
tests/endpoints/test_pages_batch.py
Normal file
299
tests/endpoints/test_pages_batch.py
Normal file
@@ -0,0 +1,299 @@
|
||||
"""Tests for Pages API batch operations."""
|
||||
|
||||
import pytest
|
||||
import responses
|
||||
|
||||
from wikijs import WikiJSClient
|
||||
from wikijs.exceptions import APIError, ValidationError
|
||||
from wikijs.models import Page, PageCreate, PageUpdate
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
"""Create a test client."""
|
||||
return WikiJSClient("https://wiki.example.com", auth="test-api-key")
|
||||
|
||||
|
||||
class TestPagesCreateMany:
|
||||
"""Tests for pages.create_many() method."""
|
||||
|
||||
@responses.activate
|
||||
def test_create_many_success(self, client):
|
||||
"""Test successful batch page creation."""
|
||||
# Mock API responses for each create
|
||||
for i in range(1, 4):
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"https://wiki.example.com/graphql",
|
||||
json={
|
||||
"data": {
|
||||
"pages": {
|
||||
"create": {
|
||||
"responseResult": {"succeeded": True},
|
||||
"page": {
|
||||
"id": i,
|
||||
"title": f"Page {i}",
|
||||
"path": f"page-{i}",
|
||||
"content": f"Content {i}",
|
||||
"description": "",
|
||||
"isPublished": True,
|
||||
"isPrivate": False,
|
||||
"tags": [],
|
||||
"locale": "en",
|
||||
"authorId": 1,
|
||||
"authorName": "Admin",
|
||||
"authorEmail": "admin@example.com",
|
||||
"editor": "markdown",
|
||||
"createdAt": "2025-01-01T00:00:00.000Z",
|
||||
"updatedAt": "2025-01-01T00:00:00.000Z",
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
status=200,
|
||||
)
|
||||
|
||||
pages_data = [
|
||||
PageCreate(title=f"Page {i}", path=f"page-{i}", content=f"Content {i}")
|
||||
for i in range(1, 4)
|
||||
]
|
||||
|
||||
created_pages = client.pages.create_many(pages_data)
|
||||
|
||||
assert len(created_pages) == 3
|
||||
for i, page in enumerate(created_pages, 1):
|
||||
assert page.id == i
|
||||
assert page.title == f"Page {i}"
|
||||
|
||||
def test_create_many_empty_list(self, client):
|
||||
"""Test create_many with empty list."""
|
||||
result = client.pages.create_many([])
|
||||
assert result == []
|
||||
|
||||
@responses.activate
|
||||
def test_create_many_partial_failure(self, client):
|
||||
"""Test create_many with some failures."""
|
||||
# Mock successful creation for first page
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"https://wiki.example.com/graphql",
|
||||
json={
|
||||
"data": {
|
||||
"pages": {
|
||||
"create": {
|
||||
"responseResult": {"succeeded": True},
|
||||
"page": {
|
||||
"id": 1,
|
||||
"title": "Page 1",
|
||||
"path": "page-1",
|
||||
"content": "Content 1",
|
||||
"description": "",
|
||||
"isPublished": True,
|
||||
"isPrivate": False,
|
||||
"tags": [],
|
||||
"locale": "en",
|
||||
"authorId": 1,
|
||||
"authorName": "Admin",
|
||||
"authorEmail": "admin@example.com",
|
||||
"editor": "markdown",
|
||||
"createdAt": "2025-01-01T00:00:00.000Z",
|
||||
"updatedAt": "2025-01-01T00:00:00.000Z",
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
status=200,
|
||||
)
|
||||
|
||||
# Mock failure for second page
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"https://wiki.example.com/graphql",
|
||||
json={"errors": [{"message": "Page already exists"}]},
|
||||
status=200,
|
||||
)
|
||||
|
||||
pages_data = [
|
||||
PageCreate(title="Page 1", path="page-1", content="Content 1"),
|
||||
PageCreate(title="Page 2", path="page-2", content="Content 2"),
|
||||
]
|
||||
|
||||
with pytest.raises(APIError) as exc_info:
|
||||
client.pages.create_many(pages_data)
|
||||
|
||||
assert "Failed to create 1/2 pages" in str(exc_info.value)
|
||||
assert "Successfully created: 1" in str(exc_info.value)
|
||||
|
||||
|
||||
class TestPagesUpdateMany:
|
||||
"""Tests for pages.update_many() method."""
|
||||
|
||||
@responses.activate
|
||||
def test_update_many_success(self, client):
|
||||
"""Test successful batch page updates."""
|
||||
# Mock API responses for each update
|
||||
for i in range(1, 4):
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"https://wiki.example.com/graphql",
|
||||
json={
|
||||
"data": {
|
||||
"updatePage": {
|
||||
"id": i,
|
||||
"title": f"Updated Page {i}",
|
||||
"path": f"page-{i}",
|
||||
"content": f"Updated Content {i}",
|
||||
"description": "",
|
||||
"isPublished": True,
|
||||
"isPrivate": False,
|
||||
"tags": [],
|
||||
"locale": "en",
|
||||
"authorId": 1,
|
||||
"authorName": "Admin",
|
||||
"authorEmail": "admin@example.com",
|
||||
"editor": "markdown",
|
||||
"createdAt": "2025-01-01T00:00:00.000Z",
|
||||
"updatedAt": "2025-01-01T00:10:00.000Z",
|
||||
}
|
||||
}
|
||||
},
|
||||
status=200,
|
||||
)
|
||||
|
||||
updates = [
|
||||
{"id": i, "content": f"Updated Content {i}", "title": f"Updated Page {i}"}
|
||||
for i in range(1, 4)
|
||||
]
|
||||
|
||||
updated_pages = client.pages.update_many(updates)
|
||||
|
||||
assert len(updated_pages) == 3
|
||||
for i, page in enumerate(updated_pages, 1):
|
||||
assert page.id == i
|
||||
assert page.title == f"Updated Page {i}"
|
||||
assert page.content == f"Updated Content {i}"
|
||||
|
||||
def test_update_many_empty_list(self, client):
|
||||
"""Test update_many with empty list."""
|
||||
result = client.pages.update_many([])
|
||||
assert result == []
|
||||
|
||||
def test_update_many_missing_id(self, client):
|
||||
"""Test update_many with missing id field."""
|
||||
updates = [{"content": "New content"}] # Missing 'id'
|
||||
|
||||
with pytest.raises(APIError) as exc_info:
|
||||
client.pages.update_many(updates)
|
||||
|
||||
assert "must have an 'id' field" in str(exc_info.value)
|
||||
|
||||
@responses.activate
|
||||
def test_update_many_partial_failure(self, client):
|
||||
"""Test update_many with some failures."""
|
||||
# Mock successful update for first page
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"https://wiki.example.com/graphql",
|
||||
json={
|
||||
"data": {
|
||||
"updatePage": {
|
||||
"id": 1,
|
||||
"title": "Updated Page 1",
|
||||
"path": "page-1",
|
||||
"content": "Updated Content 1",
|
||||
"description": "",
|
||||
"isPublished": True,
|
||||
"isPrivate": False,
|
||||
"tags": [],
|
||||
"locale": "en",
|
||||
"authorId": 1,
|
||||
"authorName": "Admin",
|
||||
"authorEmail": "admin@example.com",
|
||||
"editor": "markdown",
|
||||
"createdAt": "2025-01-01T00:00:00.000Z",
|
||||
"updatedAt": "2025-01-01T00:10:00.000Z",
|
||||
}
|
||||
}
|
||||
},
|
||||
status=200,
|
||||
)
|
||||
|
||||
# Mock failure for second page
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"https://wiki.example.com/graphql",
|
||||
json={"errors": [{"message": "Page not found"}]},
|
||||
status=200,
|
||||
)
|
||||
|
||||
updates = [
|
||||
{"id": 1, "content": "Updated Content 1"},
|
||||
{"id": 999, "content": "Updated Content 999"},
|
||||
]
|
||||
|
||||
with pytest.raises(APIError) as exc_info:
|
||||
client.pages.update_many(updates)
|
||||
|
||||
assert "Failed to update 1/2 pages" in str(exc_info.value)
|
||||
|
||||
|
||||
class TestPagesDeleteMany:
|
||||
"""Tests for pages.delete_many() method."""
|
||||
|
||||
@responses.activate
|
||||
def test_delete_many_success(self, client):
|
||||
"""Test successful batch page deletions."""
|
||||
# Mock API responses for each delete
|
||||
for i in range(1, 4):
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"https://wiki.example.com/graphql",
|
||||
json={"data": {"deletePage": {"success": True}}},
|
||||
status=200,
|
||||
)
|
||||
|
||||
result = client.pages.delete_many([1, 2, 3])
|
||||
|
||||
assert result["successful"] == 3
|
||||
assert result["failed"] == 0
|
||||
assert result["errors"] == []
|
||||
|
||||
def test_delete_many_empty_list(self, client):
|
||||
"""Test delete_many with empty list."""
|
||||
result = client.pages.delete_many([])
|
||||
assert result["successful"] == 0
|
||||
assert result["failed"] == 0
|
||||
assert result["errors"] == []
|
||||
|
||||
@responses.activate
|
||||
def test_delete_many_partial_failure(self, client):
|
||||
"""Test delete_many with some failures."""
|
||||
# Mock successful deletion for first two pages
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"https://wiki.example.com/graphql",
|
||||
json={"data": {"deletePage": {"success": True}}},
|
||||
status=200,
|
||||
)
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"https://wiki.example.com/graphql",
|
||||
json={"data": {"deletePage": {"success": True}}},
|
||||
status=200,
|
||||
)
|
||||
|
||||
# Mock failure for third page
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"https://wiki.example.com/graphql",
|
||||
json={"errors": [{"message": "Page not found"}]},
|
||||
status=200,
|
||||
)
|
||||
|
||||
with pytest.raises(APIError) as exc_info:
|
||||
client.pages.delete_many([1, 2, 999])
|
||||
|
||||
assert "Failed to delete 1/3 pages" in str(exc_info.value)
|
||||
assert "Successfully deleted: 2" in str(exc_info.value)
|
||||
233
tests/test_cache.py
Normal file
233
tests/test_cache.py
Normal file
@@ -0,0 +1,233 @@
|
||||
"""Tests for caching module."""
|
||||
|
||||
import time
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from wikijs.cache import CacheKey, MemoryCache
|
||||
from wikijs.models import Page
|
||||
|
||||
|
||||
class TestCacheKey:
|
||||
"""Tests for CacheKey class."""
|
||||
|
||||
def test_cache_key_to_string_basic(self):
|
||||
"""Test basic cache key string generation."""
|
||||
key = CacheKey("page", "123", "get")
|
||||
assert key.to_string() == "page:123:get"
|
||||
|
||||
def test_cache_key_to_string_with_params(self):
|
||||
"""Test cache key string with parameters."""
|
||||
key = CacheKey("page", "123", "list", "locale=en&tags=api")
|
||||
assert key.to_string() == "page:123:list:locale=en&tags=api"
|
||||
|
||||
def test_cache_key_different_resource_types(self):
|
||||
"""Test cache keys for different resource types."""
|
||||
page_key = CacheKey("page", "1", "get")
|
||||
user_key = CacheKey("user", "1", "get")
|
||||
assert page_key.to_string() != user_key.to_string()
|
||||
|
||||
|
||||
class TestMemoryCache:
|
||||
"""Tests for MemoryCache class."""
|
||||
|
||||
def test_init_default_values(self):
|
||||
"""Test cache initialization with default values."""
|
||||
cache = MemoryCache()
|
||||
assert cache.ttl == 300
|
||||
assert cache.max_size == 1000
|
||||
|
||||
def test_init_custom_values(self):
|
||||
"""Test cache initialization with custom values."""
|
||||
cache = MemoryCache(ttl=600, max_size=500)
|
||||
assert cache.ttl == 600
|
||||
assert cache.max_size == 500
|
||||
|
||||
def test_set_and_get(self):
|
||||
"""Test setting and getting cache values."""
|
||||
cache = MemoryCache(ttl=10)
|
||||
key = CacheKey("page", "123", "get")
|
||||
value = {"id": 123, "title": "Test Page"}
|
||||
|
||||
cache.set(key, value)
|
||||
cached = cache.get(key)
|
||||
|
||||
assert cached == value
|
||||
|
||||
def test_get_nonexistent_key(self):
|
||||
"""Test getting a key that doesn't exist."""
|
||||
cache = MemoryCache()
|
||||
key = CacheKey("page", "999", "get")
|
||||
assert cache.get(key) is None
|
||||
|
||||
def test_ttl_expiration(self):
|
||||
"""Test that cache entries expire after TTL."""
|
||||
cache = MemoryCache(ttl=1) # 1 second TTL
|
||||
key = CacheKey("page", "123", "get")
|
||||
value = {"id": 123, "title": "Test Page"}
|
||||
|
||||
cache.set(key, value)
|
||||
assert cache.get(key) == value
|
||||
|
||||
# Wait for expiration
|
||||
time.sleep(1.1)
|
||||
assert cache.get(key) is None
|
||||
|
||||
def test_lru_eviction(self):
|
||||
"""Test LRU eviction when max_size is reached."""
|
||||
cache = MemoryCache(ttl=300, max_size=3)
|
||||
|
||||
# Add 3 items
|
||||
for i in range(1, 4):
|
||||
key = CacheKey("page", str(i), "get")
|
||||
cache.set(key, {"id": i})
|
||||
|
||||
# All 3 should be present
|
||||
assert cache.get(CacheKey("page", "1", "get")) is not None
|
||||
assert cache.get(CacheKey("page", "2", "get")) is not None
|
||||
assert cache.get(CacheKey("page", "3", "get")) is not None
|
||||
|
||||
# Add 4th item - should evict oldest (1)
|
||||
cache.set(CacheKey("page", "4", "get"), {"id": 4})
|
||||
|
||||
# Item 1 should be evicted
|
||||
assert cache.get(CacheKey("page", "1", "get")) is None
|
||||
# Others should still be present
|
||||
assert cache.get(CacheKey("page", "2", "get")) is not None
|
||||
assert cache.get(CacheKey("page", "3", "get")) is not None
|
||||
assert cache.get(CacheKey("page", "4", "get")) is not None
|
||||
|
||||
def test_lru_access_updates_order(self):
|
||||
"""Test that accessing an item updates LRU order."""
|
||||
cache = MemoryCache(ttl=300, max_size=3)
|
||||
|
||||
# Add 3 items
|
||||
for i in range(1, 4):
|
||||
cache.set(CacheKey("page", str(i), "get"), {"id": i})
|
||||
|
||||
# Access item 1 (makes it most recent)
|
||||
cache.get(CacheKey("page", "1", "get"))
|
||||
|
||||
# Add 4th item - should evict item 2 (oldest now)
|
||||
cache.set(CacheKey("page", "4", "get"), {"id": 4})
|
||||
|
||||
# Item 1 should still be present (was accessed)
|
||||
assert cache.get(CacheKey("page", "1", "get")) is not None
|
||||
# Item 2 should be evicted
|
||||
assert cache.get(CacheKey("page", "2", "get")) is None
|
||||
|
||||
def test_delete(self):
|
||||
"""Test deleting cache entries."""
|
||||
cache = MemoryCache()
|
||||
key = CacheKey("page", "123", "get")
|
||||
cache.set(key, {"id": 123})
|
||||
|
||||
assert cache.get(key) is not None
|
||||
cache.delete(key)
|
||||
assert cache.get(key) is None
|
||||
|
||||
def test_clear(self):
|
||||
"""Test clearing all cache entries."""
|
||||
cache = MemoryCache()
|
||||
|
||||
# Add multiple items
|
||||
for i in range(5):
|
||||
cache.set(CacheKey("page", str(i), "get"), {"id": i})
|
||||
|
||||
# Clear cache
|
||||
cache.clear()
|
||||
|
||||
# All items should be gone
|
||||
for i in range(5):
|
||||
assert cache.get(CacheKey("page", str(i), "get")) is None
|
||||
|
||||
def test_invalidate_resource_specific(self):
|
||||
"""Test invalidating a specific resource."""
|
||||
cache = MemoryCache()
|
||||
|
||||
# Add multiple pages
|
||||
for i in range(1, 4):
|
||||
cache.set(CacheKey("page", str(i), "get"), {"id": i})
|
||||
|
||||
# Invalidate page 2
|
||||
cache.invalidate_resource("page", "2")
|
||||
|
||||
# Page 2 should be gone
|
||||
assert cache.get(CacheKey("page", "2", "get")) is None
|
||||
# Others should remain
|
||||
assert cache.get(CacheKey("page", "1", "get")) is not None
|
||||
assert cache.get(CacheKey("page", "3", "get")) is not None
|
||||
|
||||
def test_invalidate_resource_all(self):
|
||||
"""Test invalidating all resources of a type."""
|
||||
cache = MemoryCache()
|
||||
|
||||
# Add multiple pages and a user
|
||||
for i in range(1, 4):
|
||||
cache.set(CacheKey("page", str(i), "get"), {"id": i})
|
||||
cache.set(CacheKey("user", "1", "get"), {"id": 1})
|
||||
|
||||
# Invalidate all pages
|
||||
cache.invalidate_resource("page")
|
||||
|
||||
# All pages should be gone
|
||||
for i in range(1, 4):
|
||||
assert cache.get(CacheKey("page", str(i), "get")) is None
|
||||
|
||||
# User should remain
|
||||
assert cache.get(CacheKey("user", "1", "get")) is not None
|
||||
|
||||
def test_get_stats(self):
|
||||
"""Test getting cache statistics."""
|
||||
cache = MemoryCache(ttl=300, max_size=1000)
|
||||
|
||||
# Initially empty
|
||||
stats = cache.get_stats()
|
||||
assert stats["ttl"] == 300
|
||||
assert stats["max_size"] == 1000
|
||||
assert stats["current_size"] == 0
|
||||
assert stats["hits"] == 0
|
||||
assert stats["misses"] == 0
|
||||
|
||||
# Add item and access it
|
||||
key = CacheKey("page", "123", "get")
|
||||
cache.set(key, {"id": 123})
|
||||
cache.get(key) # Hit
|
||||
cache.get(CacheKey("page", "999", "get")) # Miss
|
||||
|
||||
stats = cache.get_stats()
|
||||
assert stats["current_size"] == 1
|
||||
assert stats["hits"] == 1
|
||||
assert stats["misses"] == 1
|
||||
assert "hit_rate" in stats
|
||||
|
||||
def test_cleanup_expired(self):
|
||||
"""Test cleanup of expired entries."""
|
||||
cache = MemoryCache(ttl=1)
|
||||
|
||||
# Add items
|
||||
for i in range(3):
|
||||
cache.set(CacheKey("page", str(i), "get"), {"id": i})
|
||||
|
||||
assert cache.get_stats()["current_size"] == 3
|
||||
|
||||
# Wait for expiration
|
||||
time.sleep(1.1)
|
||||
|
||||
# Run cleanup
|
||||
removed = cache.cleanup_expired()
|
||||
|
||||
assert removed == 3
|
||||
assert cache.get_stats()["current_size"] == 0
|
||||
|
||||
def test_set_updates_existing(self):
|
||||
"""Test that setting an existing key updates the value."""
|
||||
cache = MemoryCache()
|
||||
key = CacheKey("page", "123", "get")
|
||||
|
||||
cache.set(key, {"id": 123, "title": "Original"})
|
||||
assert cache.get(key)["title"] == "Original"
|
||||
|
||||
cache.set(key, {"id": 123, "title": "Updated"})
|
||||
assert cache.get(key)["title"] == "Updated"
|
||||
Reference in New Issue
Block a user