From e0ceb68f15288e650b69deb29d0fbaf447d10b9d Mon Sep 17 00:00:00 2001 From: Seraph91P Date: Sat, 24 Jan 2026 13:31:27 +0100 Subject: [PATCH 01/76] fix: Resolve TypeScript compilation errors - Remove unused imports (Download, formatDistanceToNow, Plus, Edit, CheckCircle, XCircle, useState, format) - Fix api import in Storage.tsx (named -> default import) - Fix NodeJS.Timeout type to ReturnType - Fix Zustand set callback type issue in websocket.ts - Remove unused variables (showAddTarget, queryClient) --- .gitignore | 5 +++++ frontend/src/pages/Backups.tsx | 3 +-- frontend/src/pages/Containers.tsx | 4 +--- frontend/src/pages/Retention.tsx | 1 - frontend/src/pages/Schedules.tsx | 2 +- frontend/src/pages/Storage.tsx | 2 +- frontend/src/pages/Targets.tsx | 3 +-- frontend/src/store/websocket.ts | 11 ++++++----- 8 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 6fc1afa..aa1fd9a 100644 --- a/.gitignore +++ b/.gitignore @@ -75,3 +75,8 @@ Thumbs.db # Backups (local dev) backups/ + +# git +.github/agents +.github/instructions +.github/prompts diff --git a/frontend/src/pages/Backups.tsx b/frontend/src/pages/Backups.tsx index bdab76b..d632853 100644 --- a/frontend/src/pages/Backups.tsx +++ b/frontend/src/pages/Backups.tsx @@ -7,10 +7,9 @@ import { Loader2, Trash2, RotateCcw, - Download, } from 'lucide-react' import { backupsApi, Backup } from '../api' -import { formatDistanceToNow, format } from 'date-fns' +import { format } from 'date-fns' import { de } from 'date-fns/locale' import toast from 'react-hot-toast' import { useWebSocketStore } from '../store/websocket' diff --git a/frontend/src/pages/Containers.tsx b/frontend/src/pages/Containers.tsx index 117fd3c..16430df 100644 --- a/frontend/src/pages/Containers.tsx +++ b/frontend/src/pages/Containers.tsx @@ -6,7 +6,7 @@ import { useState } from 'react' function ContainerCard({ container }: { container: Container }) { const queryClient = useQueryClient() - const [showAddTarget, setShowAddTarget] = useState(false) + const [, setShowAddTarget] = useState(false) const stopMutation = useMutation({ mutationFn: () => dockerApi.stopContainer(container.id), @@ -128,8 +128,6 @@ function ContainerCard({ container }: { container: Container }) { } export default function Containers() { - const queryClient = useQueryClient() - const { data: containers, isLoading, refetch } = useQuery({ queryKey: ['containers'], queryFn: () => dockerApi.listContainers().then((r) => r.data), diff --git a/frontend/src/pages/Retention.tsx b/frontend/src/pages/Retention.tsx index e5e965f..e78b82f 100644 --- a/frontend/src/pages/Retention.tsx +++ b/frontend/src/pages/Retention.tsx @@ -278,7 +278,6 @@ function CreatePolicyForm({ onClose }: { onClose: () => void }) { } export default function Retention() { - const queryClient = useQueryClient() const [showCreate, setShowCreate] = useState(false) const { data: policies, isLoading } = useQuery({ diff --git a/frontend/src/pages/Schedules.tsx b/frontend/src/pages/Schedules.tsx index 044da11..722e748 100644 --- a/frontend/src/pages/Schedules.tsx +++ b/frontend/src/pages/Schedules.tsx @@ -1,7 +1,7 @@ import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' import { Clock, Play, HelpCircle } from 'lucide-react' import { schedulesApi, Schedule } from '../api' -import { formatDistanceToNow, format } from 'date-fns' +import { formatDistanceToNow } from 'date-fns' import { de } from 'date-fns/locale' import toast from 'react-hot-toast' import { useState } from 'react' diff --git a/frontend/src/pages/Storage.tsx b/frontend/src/pages/Storage.tsx index c6d31ad..aa48183 100644 --- a/frontend/src/pages/Storage.tsx +++ b/frontend/src/pages/Storage.tsx @@ -17,7 +17,7 @@ import { Globe, Database, } from 'lucide-react'; -import { api } from '../api'; +import api from '../api'; interface RemoteStorage { id: number; diff --git a/frontend/src/pages/Targets.tsx b/frontend/src/pages/Targets.tsx index ef279b6..c619c8c 100644 --- a/frontend/src/pages/Targets.tsx +++ b/frontend/src/pages/Targets.tsx @@ -1,8 +1,7 @@ import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' -import { Target, Plus, Edit, Trash2, Play, Clock, CheckCircle, XCircle } from 'lucide-react' +import { Target, Trash2, Play, Clock } from 'lucide-react' import { targetsApi, backupsApi, BackupTarget } from '../api' import toast from 'react-hot-toast' -import { useState } from 'react' function TargetCard({ target }: { target: BackupTarget }) { const queryClient = useQueryClient() diff --git a/frontend/src/store/websocket.ts b/frontend/src/store/websocket.ts index af659e3..b6763a2 100644 --- a/frontend/src/store/websocket.ts +++ b/frontend/src/store/websocket.ts @@ -18,7 +18,7 @@ interface WebSocketStore { } let ws: WebSocket | null = null -let reconnectTimeout: NodeJS.Timeout | null = null +let reconnectTimeout: ReturnType | null = null export const useWebSocketStore = create((set, get) => ({ connected: false, @@ -108,11 +108,12 @@ function handleMessage( case 'backup_completed': case 'backup_failed': // Remove from progress tracking - set((state) => { - const newProgress = new Map(state.backupProgress) + { + const currentState = get() + const newProgress = new Map(currentState.backupProgress) newProgress.delete(data.backup_id as number) - return { backupProgress: newProgress } - }) + set({ backupProgress: newProgress }) + } break default: From 6e14dcd1e533e1979eff179b973565543815ee0c Mon Sep 17 00:00:00 2001 From: Seraph91P Date: Sat, 24 Jan 2026 13:52:53 +0100 Subject: [PATCH 02/76] fix(security): address critical security vulnerabilities - Fix command injection in _run_hook by using subprocess_exec with shlex.split - Fix unsafe tar extraction with path traversal validation (CVE-2007-4559) - Fix SSH command injection by sanitizing paths with shlex.quote - Add path validation to restore endpoint to prevent directory traversal - Add cron expression validation using croniter in targets API - Add withCredentials to frontend axios client for proper cookie handling - Add SECRET_KEY validation with warning for insecure defaults --- backend/app/api/backups.py | 36 ++++++++++++++++++++++- backend/app/api/targets.py | 30 ++++++++++++++++++- backend/app/backup_engine.py | 55 ++++++++++++++++++++++++++++++++--- backend/app/config.py | 27 +++++++++++++++++ backend/app/remote_storage.py | 19 +++++++++--- frontend/src/api/index.ts | 2 ++ 6 files changed, 159 insertions(+), 10 deletions(-) diff --git a/backend/app/api/backups.py b/backend/app/api/backups.py index a685e6e..ea51ab5 100644 --- a/backend/app/api/backups.py +++ b/backend/app/api/backups.py @@ -3,6 +3,7 @@ """ import asyncio +import os from typing import List, Optional from fastapi import APIRouter, HTTPException, Request from pydantic import BaseModel @@ -11,6 +12,7 @@ from app.database import Backup, BackupTarget, BackupStatus, BackupType, async_session from app.backup_engine import backup_engine from app.retention import retention_manager +from app.config import settings router = APIRouter() @@ -180,7 +182,39 @@ async def create_backup(request: CreateBackupRequest): @router.post("/{backup_id}/restore") async def restore_backup(backup_id: int, request: RestoreBackupRequest): - """Restore a backup.""" + """Restore a backup. + + If target_path is provided, it must be within allowed restore directories + to prevent path traversal attacks. + """ + if request.target_path: + # Validate the target path to prevent path traversal + abs_path = os.path.abspath(request.target_path) + + # Check for path traversal attempts + if ".." in request.target_path: + raise HTTPException( + status_code=400, + detail="Invalid restore path: path traversal not allowed" + ) + + # Ensure path is within allowed restore directories + # Allow restoring to backup base path or Docker volume paths + allowed_prefixes = [ + os.path.abspath(settings.BACKUP_BASE_PATH), + "/var/lib/docker/volumes", + ] + + is_allowed = any( + abs_path.startswith(prefix) for prefix in allowed_prefixes + ) + + if not is_allowed: + raise HTTPException( + status_code=400, + detail="Invalid restore path: must be within allowed directories" + ) + success = await backup_engine.restore_backup(backup_id, request.target_path) if not success: diff --git a/backend/app/api/targets.py b/backend/app/api/targets.py index 5697290..ff3270e 100644 --- a/backend/app/api/targets.py +++ b/backend/app/api/targets.py @@ -4,15 +4,33 @@ from typing import List, Optional from fastapi import APIRouter, HTTPException, Depends -from pydantic import BaseModel +from pydantic import BaseModel, field_validator from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from croniter import croniter from app.database import BackupTarget, RetentionPolicy, get_session, async_session router = APIRouter() +def validate_cron_expression(cron_expr: Optional[str]) -> Optional[str]: + """Validate a cron expression.""" + if cron_expr is None: + return None + + # Trim whitespace + cron_expr = cron_expr.strip() + if not cron_expr: + return None + + # Validate using croniter + if not croniter.is_valid(cron_expr): + raise ValueError(f"Invalid cron expression: {cron_expr}") + + return cron_expr + + class TargetCreate(BaseModel): """Create backup target request.""" name: str @@ -29,6 +47,11 @@ class TargetCreate(BaseModel): post_backup_command: Optional[str] = None stop_container: bool = True compression_enabled: bool = True + + @field_validator('schedule_cron') + @classmethod + def validate_schedule_cron(cls, v: Optional[str]) -> Optional[str]: + return validate_cron_expression(v) class TargetUpdate(BaseModel): @@ -42,6 +65,11 @@ class TargetUpdate(BaseModel): post_backup_command: Optional[str] = None stop_container: Optional[bool] = None compression_enabled: Optional[bool] = None + + @field_validator('schedule_cron') + @classmethod + def validate_schedule_cron(cls, v: Optional[str]) -> Optional[str]: + return validate_cron_expression(v) class TargetResponse(BaseModel): diff --git a/backend/app/backup_engine.py b/backend/app/backup_engine.py index f5e16b2..c1c43b8 100644 --- a/backend/app/backup_engine.py +++ b/backend/app/backup_engine.py @@ -8,6 +8,7 @@ import hashlib import gzip import shutil +import shlex from datetime import datetime from pathlib import Path from typing import Optional, List, Dict, Any, Callable @@ -314,9 +315,22 @@ def calc(): return await loop.run_in_executor(None, calc) async def _run_hook(self, command: str): - """Run a pre/post backup hook command.""" - process = await asyncio.create_subprocess_shell( - command, + """Run a pre/post backup hook command safely. + + Uses shlex.split to parse the command into arguments, preventing + shell injection attacks by avoiding shell=True. + """ + try: + # Parse command into safe argument list + args = shlex.split(command) + except ValueError as e: + raise Exception(f"Invalid hook command syntax: {e}") + + if not args: + raise Exception("Empty hook command") + + process = await asyncio.create_subprocess_exec( + *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) @@ -421,8 +435,41 @@ async def restore_backup(self, backup_id: int, target_path: Optional[str] = None return False def _extract_tar(self, source: str, dest: str, mode: str): - """Extract tar archive.""" + """Extract tar archive safely. + + Validates that all extracted files stay within the destination + directory to prevent path traversal attacks (CVE-2007-4559). + """ + abs_dest = os.path.abspath(dest) + with tarfile.open(source, mode) as tar: + # Validate all members before extraction + for member in tar.getmembers(): + member_path = os.path.join(dest, member.name) + abs_member = os.path.abspath(member_path) + + # Check for path traversal + if not abs_member.startswith(abs_dest + os.sep) and abs_member != abs_dest: + raise ValueError( + f"Path traversal detected in archive: {member.name}" + ) + + # Check for absolute paths in archive + if os.path.isabs(member.name): + raise ValueError( + f"Absolute path detected in archive: {member.name}" + ) + + # Check for suspicious symlinks + if member.issym() or member.islnk(): + link_path = os.path.join(dest, os.path.dirname(member.name), member.linkname) + abs_link = os.path.abspath(link_path) + if not abs_link.startswith(abs_dest + os.sep): + raise ValueError( + f"Symlink escape detected in archive: {member.name} -> {member.linkname}" + ) + + # Safe to extract after validation tar.extractall(dest) async def estimate_backup_duration(self, target: BackupTarget) -> int: diff --git a/backend/app/config.py b/backend/app/config.py index 88ed3d6..014f242 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -3,8 +3,13 @@ """ from pydantic_settings import BaseSettings +from pydantic import field_validator from typing import List import os +import logging +import warnings + +logger = logging.getLogger(__name__) class Settings(BaseSettings): @@ -39,6 +44,28 @@ class Settings(BaseSettings): # Scheduling SCHEDULER_TIMEZONE: str = "Europe/Berlin" + @field_validator('SECRET_KEY') + @classmethod + def validate_secret_key(cls, v: str) -> str: + """Validate that SECRET_KEY has been changed from default.""" + insecure_defaults = [ + "change-me-in-production", + "changeme", + "secret", + "your-secret-key", + ] + + if v.lower() in insecure_defaults or len(v) < 32: + warnings.warn( + "โš ๏ธ SECURITY WARNING: SECRET_KEY is insecure! " + "Please set a strong SECRET_KEY (at least 32 characters) " + "via environment variable for production use.", + UserWarning, + stacklevel=2 + ) + + return v + class Config: env_file = ".env" case_sensitive = True diff --git a/backend/app/remote_storage.py b/backend/app/remote_storage.py index 273754a..df3be33 100644 --- a/backend/app/remote_storage.py +++ b/backend/app/remote_storage.py @@ -14,6 +14,7 @@ import os import subprocess import shutil +import shlex from abc import ABC, abstractmethod from pathlib import Path from typing import Optional, Dict, Any, List @@ -212,7 +213,9 @@ async def upload(self, local_path: Path, remote_path: str) -> bool: try: # Create remote directory first remote_dir = os.path.dirname(f"{self.config.base_path}/{remote_path}") - mkdir_cmd = self._build_ssh_command(f"mkdir -p {remote_dir}") + # Sanitize the path to prevent command injection + safe_dir = shlex.quote(remote_dir) + mkdir_cmd = self._build_ssh_command(f"mkdir -p {safe_dir}") process = await asyncio.create_subprocess_exec( *mkdir_cmd, @@ -266,7 +269,11 @@ async def download(self, remote_path: str, local_path: Path) -> bool: return False def _build_ssh_command(self, remote_cmd: str) -> List[str]: - """Build SSH command""" + """Build SSH command. + + Note: The remote_cmd should already have paths sanitized with shlex.quote() + before being passed to this method. + """ cmd = ["ssh"] if self.config.port: cmd.extend(["-p", str(self.config.port)]) @@ -281,7 +288,9 @@ def _build_ssh_command(self, remote_cmd: str) -> List[str]: async def delete(self, remote_path: str) -> bool: try: full_path = f"{self.config.base_path}/{remote_path}" - cmd = self._build_ssh_command(f"rm -f {full_path}") + # Sanitize the path to prevent command injection + safe_path = shlex.quote(full_path) + cmd = self._build_ssh_command(f"rm -f {safe_path}") process = await asyncio.create_subprocess_exec( *cmd, @@ -297,7 +306,9 @@ async def delete(self, remote_path: str) -> bool: async def list_files(self, remote_path: str = "") -> List[Dict[str, Any]]: try: full_path = f"{self.config.base_path}/{remote_path}".replace("//", "/") - cmd = self._build_ssh_command(f"ls -la {full_path}") + # Sanitize the path to prevent command injection + safe_path = shlex.quote(full_path) + cmd = self._build_ssh_command(f"ls -la {safe_path}") process = await asyncio.create_subprocess_exec( *cmd, diff --git a/frontend/src/api/index.ts b/frontend/src/api/index.ts index 4f0f90b..74e68ec 100644 --- a/frontend/src/api/index.ts +++ b/frontend/src/api/index.ts @@ -5,6 +5,8 @@ const api = axios.create({ headers: { 'Content-Type': 'application/json', }, + // Include credentials (cookies) for authentication + withCredentials: true, }) // Types From 3aee2e10684accc2e71d20a70c61a5077b639741 Mon Sep 17 00:00:00 2001 From: Seraph91P Date: Sat, 24 Jan 2026 16:33:00 +0100 Subject: [PATCH 03/76] feat: Add comprehensive testing infrastructure and backup engine improvements Backend improvements: - Add BackupMetrics class for tracking backup performance - Add pre-backup validation with validate_backup_prerequisites() - Add concurrency control with semaphore (MAX_CONCURRENT_BACKUPS) - Add API pagination for backups endpoint - Add metrics summary and validation API endpoints - Fix metadata column name conflict (metadata -> backup_metadata) Testing infrastructure: - Add pytest configuration and fixtures (conftest.py) - Add comprehensive unit tests for backup engine (752 lines) - Add database model tests - Add Docker client wrapper tests - Add scheduler tests - Add integration test script (integration_test.sh) - Add test runner script (test.sh) Frontend testing: - Add vitest configuration with coverage thresholds - Add MSW mock server and handlers - Add API layer tests - Add Backups page tests - Add Dashboard page tests - Add WebSocket store tests Documentation: - Add TESTING_GUIDE.md with comprehensive testing checklist - Add TESTING.md quick reference - Add GitHub Copilot instructions for code review, security, etc. - Add GitHub Actions test workflow All 19+ unit tests passing, integration tests verified working. --- .github/copilot-instructions.md | 84 ++ .../instructions/code-review.instructions.md | 193 +++++ .github/instructions/docker.instructions.md | 87 ++ .../documentation.instructions.md | 112 +++ .../instructions/performance.instructions.md | 148 ++++ .github/instructions/python.instructions.md | 87 ++ .../react-typescript.instructions.md | 122 +++ .github/instructions/security.instructions.md | 141 ++++ .github/instructions/testing.instructions.md | 119 +++ .github/workflows/copilot-setup-steps.yml | 94 +++ .github/workflows/release.yml | 13 +- .github/workflows/test.yml | 208 +++++ .gitignore | 1 - Dockerfile | 124 ++- README.md | 329 +++----- TESTING.md | 374 +++++++++ backend/app/api/backups.py | 87 +- backend/app/backup_engine.py | 338 +++++++- backend/app/database.py | 8 +- backend/pytest.ini | 23 + backend/requirements-dev.txt | 8 + backend/tests/__init__.py | 1 + backend/tests/conftest.py | 124 +++ backend/tests/test_api_backups.py | 545 +++++++++++++ backend/tests/test_backup_engine.py | 752 ++++++++++++++++++ backend/tests/test_database.py | 346 ++++++++ backend/tests/test_docker_client.py | 264 ++++++ backend/tests/test_scheduler.py | 422 ++++++++++ docs/TESTING_GUIDE.md | 261 ++++++ frontend/package.json | 14 +- frontend/src/api/__tests__/index.test.ts | 367 +++++++++ frontend/src/pages/__tests__/Backups.test.tsx | 295 +++++++ .../src/pages/__tests__/Dashboard.test.tsx | 348 ++++++++ .../src/store/__tests__/websocket.test.ts | 396 +++++++++ frontend/src/test/mocks/handlers.ts | 295 +++++++ frontend/src/test/mocks/server.ts | 5 + frontend/src/test/setup.ts | 56 ++ frontend/vitest.config.ts | 37 + integration_test.sh | 436 ++++++++++ test.sh | 231 ++++++ 40 files changed, 7605 insertions(+), 290 deletions(-) create mode 100644 .github/copilot-instructions.md create mode 100644 .github/instructions/code-review.instructions.md create mode 100644 .github/instructions/docker.instructions.md create mode 100644 .github/instructions/documentation.instructions.md create mode 100644 .github/instructions/performance.instructions.md create mode 100644 .github/instructions/python.instructions.md create mode 100644 .github/instructions/react-typescript.instructions.md create mode 100644 .github/instructions/security.instructions.md create mode 100644 .github/instructions/testing.instructions.md create mode 100644 .github/workflows/copilot-setup-steps.yml create mode 100644 .github/workflows/test.yml create mode 100644 TESTING.md create mode 100644 backend/pytest.ini create mode 100644 backend/requirements-dev.txt create mode 100644 backend/tests/__init__.py create mode 100644 backend/tests/conftest.py create mode 100644 backend/tests/test_api_backups.py create mode 100644 backend/tests/test_backup_engine.py create mode 100644 backend/tests/test_database.py create mode 100644 backend/tests/test_docker_client.py create mode 100644 backend/tests/test_scheduler.py create mode 100644 docs/TESTING_GUIDE.md create mode 100644 frontend/src/api/__tests__/index.test.ts create mode 100644 frontend/src/pages/__tests__/Backups.test.tsx create mode 100644 frontend/src/pages/__tests__/Dashboard.test.tsx create mode 100644 frontend/src/store/__tests__/websocket.test.ts create mode 100644 frontend/src/test/mocks/handlers.ts create mode 100644 frontend/src/test/mocks/server.ts create mode 100644 frontend/src/test/setup.ts create mode 100644 frontend/vitest.config.ts create mode 100755 integration_test.sh create mode 100644 test.sh diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..f079818 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,84 @@ +--- +description: "Main repository instructions for DockerVault - Docker Volume Backup solution" +--- + +# DockerVault Development Guidelines + +DockerVault is an automated Docker backup solution with a modern web interface, built with Python FastAPI backend and React TypeScript frontend. + +## Project Architecture + +- **Backend**: Python FastAPI with SQLAlchemy, Docker SDK, APScheduler for task scheduling +- **Frontend**: React 19 + TypeScript, Vite build system, TailwindCSS styling +- **Database**: SQLite with async support via aiosqlite +- **Infrastructure**: Docker containerized, nginx for frontend serving +- **Storage**: Local Docker volumes with remote backup support (S3, FTP, WebDAV) + +## Development Standards + +### Code Quality +- Follow language-specific guidelines in [instructions/](./instructions/) directory +- Maintain consistent naming conventions across frontend and backend +- Write comprehensive tests for both API endpoints and React components +- Use TypeScript strict mode for frontend type safety +- Follow Python PEP 8 standards with type hints + +### Architecture Principles +- Keep backend and frontend loosely coupled via REST API +- Use WebSocket for real-time backup progress updates +- Implement proper error handling and user feedback +- Design for scalability with async patterns +- Follow Docker best practices for containerization + +### Security +- Never commit sensitive configuration or credentials +- Use environment variables for all configuration +- Implement proper input validation on both frontend and backend +- Follow security best practices for Docker containers +- Use non-root users in containers + +### Testing Strategy +- Unit tests for backend services and utilities +- Integration tests for API endpoints +- Component tests for React components +- End-to-end tests for critical backup workflows +- Maintain test coverage above 80% + +### Documentation +- Keep README.md updated with setup and usage instructions +- Document API endpoints with OpenAPI/Swagger +- Add inline documentation for complex business logic +- Update deployment guides for any infrastructure changes + +## Specific Guidelines + +### Backup System +- Implement robust error handling for backup operations +- Provide clear progress indicators and status messages +- Support resumable operations where possible +- Log all backup activities with appropriate detail levels + +### Docker Integration +- Use Docker SDK for container and volume management +- Implement proper cleanup for failed operations +- Handle Docker daemon connection errors gracefully +- Support both local and remote Docker endpoints + +### File Operations +- Use async file I/O patterns for better performance +- Implement proper stream processing for large files +- Handle disk space issues and quota limits +- Support compression and encryption options + +### User Interface +- Provide intuitive backup configuration workflows +- Display real-time status and progress information +- Implement responsive design for various screen sizes +- Use consistent icons and terminology throughout + +When implementing new features, consider: +1. Impact on existing backup processes +2. Resource usage and performance implications +3. User experience and workflow efficiency +4. Error scenarios and recovery procedures +5. Documentation and help text requirements \ No newline at end of file diff --git a/.github/instructions/code-review.instructions.md b/.github/instructions/code-review.instructions.md new file mode 100644 index 0000000..f16261e --- /dev/null +++ b/.github/instructions/code-review.instructions.md @@ -0,0 +1,193 @@ +--- +applyTo: "**/*" +description: "Code review standards and GitHub review guidelines for DockerVault" +--- + +# Code Review Guidelines + +## General Code Review Principles + +- Focus on code correctness, readability, and maintainability +- Provide constructive feedback with specific suggestions +- Consider the bigger picture: architecture, design patterns, and consistency +- Review for security vulnerabilities and performance implications +- Check that tests are adequate and meaningful +- Ensure documentation is updated when necessary + +## What to Look For + +### Code Quality + +- Adherence to project coding standards and style guides +- Proper error handling and edge case coverage +- Clear and descriptive variable and function names +- Appropriate code organization and separation of concerns +- Elimination of code duplication and dead code +- Proper use of design patterns and architectural principles + +### Functionality + +- Code does what it's supposed to do +- Business logic is correct and complete +- Edge cases and error scenarios are handled +- Input validation is comprehensive +- Output formats match specifications +- Integration points work correctly + +### Security + +- Input sanitization and validation +- Proper authentication and authorization +- No hardcoded secrets or sensitive data +- Secure handling of file operations and paths +- Proper use of environment variables +- Adherence to security best practices + +### Performance + +- Efficient algorithms and data structures +- Appropriate use of caching and optimization +- Proper resource management and cleanup +- Async patterns used correctly +- No obvious performance bottlenecks +- Reasonable memory usage patterns + +### Testing + +- Adequate test coverage for new functionality +- Tests are meaningful and test the right things +- Test data is appropriate and realistic +- Mocking is used appropriately +- Tests are maintainable and not brittle +- Edge cases and error scenarios are tested + +## Language-Specific Review Points + +### Python Backend Reviews + +- Type hints are used consistently +- Async/await patterns are used properly +- Database operations use proper transaction handling +- API endpoints follow REST conventions +- Error responses are consistent and informative +- Docker SDK usage is efficient and safe + +### React Frontend Reviews + +- Components follow React best practices +- State management is appropriate (local vs global) +- TypeScript types are comprehensive and accurate +- Accessibility considerations are addressed +- Performance optimizations are appropriate +- Error boundaries handle failures gracefully + +### Docker Reviews + +- Dockerfile follows multi-stage build practices +- Security best practices are implemented +- Image size is optimized +- Resource limits are appropriate +- Health checks are implemented +- Environment configuration is externalized + +## Review Process + +### Before Submitting a PR + +- Self-review your code thoroughly +- Ensure all tests pass and coverage is adequate +- Update documentation as needed +- Write a clear PR description with context +- Link to relevant issues or requirements +- Consider the impact on existing functionality + +### Conducting a Review + +- Understand the context and requirements +- Review the code changes line by line +- Test the changes locally when appropriate +- Provide specific, actionable feedback +- Suggest alternatives when requesting changes +- Acknowledge good practices and improvements + +### Review Comments + +- Use clear, respectful language +- Explain the reasoning behind suggestions +- Provide code examples when helpful +- Use appropriate prefixes (nit, optional, blocking) +- Focus on the code, not the person +- Ask questions to understand intent when unclear + +### GitHub Review Features + +- Use GitHub's suggestion feature for small changes +- Mark conversations as resolved after addressing +- Use appropriate review status (Comment, Approve, Request Changes) +- Add reviewers with relevant expertise +- Use draft PRs for work-in-progress reviews +- Link PRs to issues and project boards + +## DockerVault-Specific Review Areas + +### Backup Operations + +- Proper error handling for backup failures +- Progress tracking and user feedback +- Resource cleanup after operations +- Validation of backup integrity +- Proper handling of large files and volumes +- Scheduling and automation logic + +### Docker Integration + +- Safe Docker socket access +- Proper container lifecycle management +- Volume mounting and permissions +- Network configuration and security +- Resource limits and monitoring +- Error handling for Docker daemon issues + +### User Interface + +- Consistent user experience across features +- Proper loading states and error messages +- Accessibility and responsive design +- Real-time updates and WebSocket handling +- Form validation and user feedback +- Navigation and routing logic + +### Configuration Management + +- Environment variable usage +- Configuration validation +- Default values and documentation +- Security of sensitive configuration +- Deployment and environment differences +- Migration and upgrade procedures + +## Common Issues to Watch For + +- Hardcoded configuration values +- Missing error handling or overly broad exception catching +- Security vulnerabilities (path traversal, injection, etc.) +- Performance issues (N+1 queries, inefficient algorithms) +- Inconsistent code style or naming conventions +- Missing or inadequate tests +- Outdated or missing documentation +- Breaking changes without proper versioning +- Resource leaks or improper cleanup +- Race conditions in concurrent code + +## Review Checklist + +- [ ] Code follows project style and conventions +- [ ] Functionality is correct and complete +- [ ] Error handling is comprehensive +- [ ] Security best practices are followed +- [ ] Performance considerations are addressed +- [ ] Tests are adequate and meaningful +- [ ] Documentation is updated as needed +- [ ] No breaking changes without proper communication +- [ ] Configuration is externalized appropriately +- [ ] Resource usage is reasonable and monitored \ No newline at end of file diff --git a/.github/instructions/docker.instructions.md b/.github/instructions/docker.instructions.md new file mode 100644 index 0000000..f85665d --- /dev/null +++ b/.github/instructions/docker.instructions.md @@ -0,0 +1,87 @@ + +--- +applyTo: "**/Dockerfile,**/docker-compose*.yml,**/docker-compose*.yaml" +description: "Docker containerization best practices for DockerVault" +--- + +# Docker Development Guidelines + +## Multi-Stage Builds + +- Use multi-stage builds to separate build and runtime dependencies +- Name build stages descriptively (AS build, AS production) +- Copy only necessary artifacts between stages +- Use different base images for build and runtime when appropriate + +## Base Image Selection + +- Use official, minimal base images (alpine variants when possible) +- Use specific version tags, avoid 'latest' in production +- Prefer language-specific official images (python:3.11-slim, node:18-alpine) +- Regularly update base images for security patches + +## Layer Optimization + +- Order instructions from least to most frequently changing +- Combine RUN commands to minimize layers +- Clean up package caches in the same RUN command +- Use .dockerignore to exclude unnecessary files from build context + +## Security Best Practices + +- Run containers as non-root user +- Create dedicated users for applications +- Use minimal base images to reduce attack surface +- Scan images for vulnerabilities regularly +- Never include secrets or credentials in image layers + +## Configuration Management + +- Use environment variables for runtime configuration +- Provide sensible defaults with ENV instructions +- Validate required environment variables at startup +- Use secrets management for sensitive data + +## Health Checks and Monitoring + +- Define HEALTHCHECK instructions for application monitoring +- Design health checks that verify actual functionality +- Use appropriate intervals and timeouts +- Implement both liveness and readiness checks + +## Volume and Data Management + +- Use named volumes for persistent data +- Never store persistent data in container's writable layer +- Implement proper backup strategies for volumes +- Use bind mounts sparingly and only for development + +## Network Configuration + +- Create custom networks for service isolation +- Use service discovery features of Docker Compose +- Implement proper network segmentation +- Document exposed ports with EXPOSE instruction + +## Resource Management + +- Set appropriate CPU and memory limits +- Monitor resource usage to tune limits +- Use resource quotas in orchestrated environments +- Implement proper logging strategies + +## Development vs Production + +- Use separate Docker Compose files for different environments +- Override configurations with environment-specific values +- Implement proper build optimization for production +- Use development-friendly settings for local development + +## DockerVault Specific Guidelines + +- Backend container should have access to Docker socket for backup operations +- Frontend should be served by nginx in production +- Use proper volume mounts for backup storage locations +- Implement proper cleanup procedures for temporary files +- Handle Docker daemon connection issues gracefully +- Use appropriate resource limits for backup operations \ No newline at end of file diff --git a/.github/instructions/documentation.instructions.md b/.github/instructions/documentation.instructions.md new file mode 100644 index 0000000..815f355 --- /dev/null +++ b/.github/instructions/documentation.instructions.md @@ -0,0 +1,112 @@ +--- +applyTo: "**/*.md,**/*.rst,**/*.txt,**/docs/**/*" +description: "Documentation requirements and standards for DockerVault" +--- + +# Documentation Guidelines + +## General Documentation Principles + +- Write documentation for your future self and new team members +- Keep documentation up-to-date with code changes +- Use clear, concise language and avoid jargon +- Include examples and practical use cases +- Structure information logically with proper headings + +## README.md Standards + +- Include project description and key features +- Provide clear installation and setup instructions +- Document prerequisites and system requirements +- Include usage examples and common workflows +- Add troubleshooting section for common issues +- Include contribution guidelines and development setup + +## API Documentation + +- Use OpenAPI/Swagger for REST API documentation +- Include request/response examples with realistic data +- Document all parameters, headers, and status codes +- Provide clear error response formats +- Include authentication and authorization details +- Add rate limiting and usage guidelines + +## Code Documentation + +- Write docstrings for all public functions and classes +- Use type hints consistently in Python code +- Add inline comments for complex business logic +- Document non-obvious implementation decisions +- Include examples in docstrings where helpful +- Explain the 'why' not just the 'what' + +## User Documentation + +- Create step-by-step guides for common tasks +- Include screenshots and visual guides where helpful +- Document configuration options and their effects +- Provide troubleshooting guides for common problems +- Include backup and recovery procedures +- Document security considerations and best practices + +## Technical Documentation + +- Document system architecture and component interactions +- Include deployment guides and infrastructure requirements +- Document database schema and migration procedures +- Provide monitoring and logging configuration guides +- Include performance tuning and optimization guides +- Document disaster recovery procedures + +## Docker and Deployment Documentation + +- Document Docker Compose setup and configuration +- Include environment variable reference +- Provide production deployment checklist +- Document backup and restore procedures for containers +- Include scaling and load balancing guidelines +- Document security hardening steps + +## Development Documentation + +- Document development environment setup +- Include coding standards and style guides +- Document testing procedures and coverage requirements +- Provide contribution guidelines and code review process +- Include build and deployment pipeline documentation +- Document debugging procedures and tools + +## Change Documentation + +- Maintain CHANGELOG.md with version history +- Document breaking changes and migration guides +- Include feature deprecation notices +- Document known issues and their workarounds +- Provide upgrade instructions between versions + +## Documentation Maintenance + +- Review documentation during code reviews +- Update documentation as part of feature development +- Regularly audit documentation for accuracy +- Remove or update outdated information +- Test documentation instructions with fresh environments +- Gather feedback from users and improve based on common questions + +## Formatting Standards + +- Use Markdown for most documentation +- Follow consistent heading structure (H1 for title, H2 for main sections) +- Use code blocks with appropriate syntax highlighting +- Include table of contents for longer documents +- Use bullet points and numbered lists appropriately +- Ensure proper spelling and grammar + +## DockerVault Specific Documentation + +- Document backup strategies and best practices +- Include volume backup and restore procedures +- Document remote storage configuration options +- Provide scheduling and automation setup guides +- Include monitoring and alerting configuration +- Document integration with external systems \ No newline at end of file diff --git a/.github/instructions/performance.instructions.md b/.github/instructions/performance.instructions.md new file mode 100644 index 0000000..07882c1 --- /dev/null +++ b/.github/instructions/performance.instructions.md @@ -0,0 +1,148 @@ +--- +applyTo: "**/*.py,**/*.ts,**/*.tsx,**/docker-compose*.yml" +description: "Performance optimization guidelines for DockerVault" +--- + +# Performance Guidelines + +## General Performance Principles + +- Optimize for the most common use cases +- Measure before optimizing - use profiling tools +- Consider memory usage alongside CPU performance +- Implement proper caching strategies +- Use async patterns for I/O-bound operations +- Monitor performance metrics in production + +## Backend Performance (Python/FastAPI) + +- Use async/await for all I/O operations +- Implement connection pooling for databases and external services +- Use streaming for large file operations +- Cache frequently accessed data with appropriate TTL +- Use background tasks for long-running operations +- Implement proper pagination for large data sets + +### Database Performance + +- Use database indexes on frequently queried columns +- Implement proper query optimization +- Use connection pooling and connection reuse +- Batch database operations where possible +- Monitor query execution times and optimize slow queries +- Use database-specific optimizations for SQLite + +### File Operations + +- Use async file I/O for better concurrency +- Implement streaming for large files to manage memory usage +- Use compression to reduce storage and transfer overhead +- Implement proper progress tracking for long operations +- Use parallel processing for independent file operations +- Optimize temporary file usage and cleanup + +### Docker Operations + +- Use Docker SDK efficiently with proper connection management +- Implement batching for multiple container operations +- Monitor Docker daemon resource usage +- Use appropriate timeouts for Docker operations +- Implement proper cleanup to prevent resource leaks +- Cache Docker image information where appropriate + +## Frontend Performance (React/TypeScript) + +- Use React.memo for expensive components +- Implement code splitting with React.lazy +- Optimize bundle size with tree shaking +- Use proper dependency arrays in hooks +- Implement virtual scrolling for large lists +- Optimize re-renders with useMemo and useCallback + +### State Management Performance + +- Use React Query for efficient server state caching +- Implement proper cache invalidation strategies +- Use optimistic updates for better perceived performance +- Minimize unnecessary state updates and re-renders +- Use Zustand selectors to prevent unnecessary subscriptions + +### Network Performance + +- Implement request deduplication +- Use proper HTTP caching headers +- Implement retry logic with exponential backoff +- Batch API requests where possible +- Use WebSocket efficiently for real-time updates +- Implement offline support and sync strategies + +### UI Performance + +- Implement lazy loading for images and components +- Use proper loading states and skeleton screens +- Optimize CSS and avoid layout thrashing + - Implement proper error boundaries to prevent cascading failures +- Use debouncing for user input handling + +## Docker Performance + +- Use multi-stage builds to reduce image size +- Optimize layer caching for faster builds +- Use appropriate resource limits (CPU, memory) +- Implement proper health checks with reasonable intervals +- Use volumes for persistent data to avoid copy overhead +- Optimize Docker networking for service communication + +## System Performance + +- Monitor system resources (CPU, memory, disk, network) +- Implement proper logging that doesn't impact performance +- Use appropriate log levels and rotation policies +- Monitor backup operation performance and resource usage +- Implement alerting for performance degradation +- Use appropriate scheduling for background operations + +## Backup Performance + +- Implement incremental backup strategies +- Use compression to reduce storage and network overhead +- Implement parallel processing for independent backup tasks +- Monitor backup duration and optimize bottlenecks +- Use appropriate chunk sizes for large file operations +- Implement resume capability for interrupted operations + +## Remote Storage Performance + +- Use connection pooling for remote storage operations +- Implement proper retry mechanisms with backoff +- Use multipart uploads for large files +- Implement progress tracking and cancellation +- Optimize network usage with compression and deduplication +- Cache remote storage metadata appropriately + +## Monitoring and Profiling + +- Use application performance monitoring (APM) tools +- Implement custom metrics for business-critical operations +- Monitor resource usage patterns over time +- Use profiling tools to identify performance bottlenecks +- Implement performance regression testing +- Set up alerting for performance threshold violations + +## Scalability Considerations + +- Design for horizontal scaling where possible +- Use stateless application design patterns +- Implement proper load balancing strategies +- Use appropriate queueing for background tasks +- Plan for data growth and archival strategies +- Consider distributed caching solutions for scale + +## Development Performance + +- Use development tools that support hot reloading +- Optimize build times with proper caching +- Use appropriate test strategies to minimize test execution time +- Implement parallel test execution where possible +- Use efficient development environment setup +- Monitor and optimize CI/CD pipeline performance \ No newline at end of file diff --git a/.github/instructions/python.instructions.md b/.github/instructions/python.instructions.md new file mode 100644 index 0000000..c7616aa --- /dev/null +++ b/.github/instructions/python.instructions.md @@ -0,0 +1,87 @@ + +--- +applyTo: "**/*.py" +description: "Python development standards for DockerVault backend" +--- + +# Python Development Guidelines + +## Code Style and Formatting + +- Follow **PEP 8** style guide for Python +- Use 4 spaces for indentation +- Keep lines under 88 characters (Black formatter standard) +- Use type hints for all function parameters and return values +- Write clear and concise docstrings following PEP 257 conventions + +## FastAPI Specific Guidelines + +- Use Pydantic models for request/response validation +- Implement proper dependency injection patterns +- Use async/await for all I/O operations +- Define clear API endpoint groupings with routers +- Include comprehensive OpenAPI documentation with examples + +## Database and SQLAlchemy + +- Use async SQLAlchemy patterns with aiosqlite +- Define clear database models with proper relationships +- Implement database migrations for schema changes +- Use connection pooling appropriately +- Handle database errors gracefully with proper rollbacks + +## Docker SDK Integration + +- Use async patterns with aiodocker or docker-py +- Implement proper connection management and cleanup +- Handle Docker daemon unavailability scenarios +- Use context managers for resource management +- Log Docker operations with appropriate detail levels + +## Error Handling and Logging + +- Use structured logging with JSON format for production +- Implement custom exception classes for domain-specific errors +- Log all backup operations with trace IDs for debugging +- Handle edge cases like disk space, permissions, and network issues +- Provide meaningful error messages for API responses + +## Async Programming + +- Use asyncio properly for concurrent operations +- Implement proper cancellation handling for long-running tasks +- Use semaphores and rate limiting for external API calls +- Handle backpressure in streaming operations +- Test async code with pytest-asyncio + +## Testing Guidelines + +- Write unit tests for all business logic functions +- Use pytest fixtures for database and Docker test setup +- Mock external dependencies (Docker daemon, remote storage) +- Test error scenarios and edge cases +- Include performance tests for file operations + +## Security Considerations + +- Validate all input data with Pydantic models +- Use environment variables for sensitive configuration +- Implement proper authentication and authorization +- Sanitize file paths to prevent directory traversal +- Use secure defaults for all configuration options + +## Performance Optimization + +- Use connection pooling for database operations +- Implement streaming for large file operations +- Use background tasks for long-running operations +- Cache frequently accessed data appropriately +- Profile code to identify performance bottlenecks + +## Code Organization + +- Separate business logic from API endpoints +- Use dependency injection for testability +- Group related functionality in modules +- Keep configuration in separate files +- Follow domain-driven design principles \ No newline at end of file diff --git a/.github/instructions/react-typescript.instructions.md b/.github/instructions/react-typescript.instructions.md new file mode 100644 index 0000000..f64c75d --- /dev/null +++ b/.github/instructions/react-typescript.instructions.md @@ -0,0 +1,122 @@ + +--- +applyTo: "**/*.tsx,**/*.ts,**/*.jsx,**/*.js,frontend/**/*.css" +description: "React TypeScript development standards for DockerVault frontend" +--- + +# React TypeScript Development Guidelines + +## Project Context + +- React 19+ with TypeScript for type safety +- Vite for fast development and optimized builds +- TailwindCSS for utility-first styling +- React Query (TanStack Query) for server state management +- Zustand for client state management + +## Component Architecture + +- Use functional components with hooks as the primary pattern +- Implement component composition over inheritance +- Organize components by feature or domain for scalability +- Separate presentational and container components clearly +- Use custom hooks for reusable stateful logic +- Keep components small and focused on a single concern + +## TypeScript Integration + +- Use TypeScript interfaces for props, state, and API responses +- Define proper types for event handlers and refs +- Use strict mode in tsconfig.json for maximum type safety +- Leverage React's built-in types (React.FC, React.ComponentProps) +- Create union types for component variants and states +- Use generic types for reusable components + +## State Management + +- Use React Query for server state and caching +- Use Zustand for global client state (UI state, user preferences) +- Use useState for local component state +- Implement useReducer for complex local state logic +- Use useContext sparingly for theme/auth context + +## Styling with TailwindCSS + +- Use Tailwind utility classes for consistent styling +- Create reusable component classes for common patterns +- Implement responsive design with mobile-first approach +- Use Tailwind's color system and spacing scale consistently +- Combine with clsx/tailwind-merge for conditional classes + +## Performance Optimization + +- Use React.memo for component memoization when appropriate +- Implement code splitting with React.lazy and Suspense +- Use useMemo and useCallback judiciously to prevent unnecessary re-renders +- Optimize bundle size with tree shaking and dynamic imports +- Implement virtual scrolling for large data lists + +## Data Fetching with React Query + +- Use React Query for all server state management +- Implement proper loading, error, and success states +- Use optimistic updates for better user experience +- Implement proper caching strategies with stale-while-revalidate +- Handle offline scenarios and network errors gracefully + +## Error Handling + +- Implement Error Boundaries for component-level error handling +- Use proper error states in data fetching +- Provide meaningful error messages to users +- Log errors appropriately for debugging +- Handle async errors in effects and event handlers + +## Form Handling + +- Use controlled components for form inputs +- Implement proper form validation with TypeScript types +- Handle form submission and error states appropriately +- Use React Hook Form for complex forms +- Implement accessibility features for forms (labels, ARIA attributes) + +## Testing Guidelines + +- Write component tests using React Testing Library +- Test component behavior, not implementation details +- Mock external dependencies and API calls appropriately +- Test accessibility features and keyboard navigation +- Use MSW (Mock Service Worker) for API mocking + +## Real-time Features + +- Use WebSocket connections for backup progress updates +- Implement proper connection management and reconnection +- Handle WebSocket errors and connection states +- Update UI reactively based on real-time data +- Provide visual indicators for connection status + +## Security Considerations + +- Sanitize user inputs to prevent XSS attacks +- Validate and escape data before rendering +- Use HTTPS for all API communications +- Implement proper authentication state management +- Avoid storing sensitive data in localStorage + +## Accessibility + +- Use semantic HTML elements appropriately +- Implement proper ARIA attributes and roles +- Ensure keyboard navigation works for all interactive elements +- Provide alt text for images and descriptive text for icons +- Implement proper color contrast ratios +- Test with screen readers + +## Code Organization + +- Organize components by feature rather than type +- Use index files for clean imports +- Separate API calls into service files +- Keep constants and types in separate files +- Use absolute imports for cleaner import paths \ No newline at end of file diff --git a/.github/instructions/security.instructions.md b/.github/instructions/security.instructions.md new file mode 100644 index 0000000..7118d1a --- /dev/null +++ b/.github/instructions/security.instructions.md @@ -0,0 +1,141 @@ +--- +applyTo: "**/*" +description: "Security best practices and requirements for DockerVault" +--- + +# Security Guidelines + +## General Security Principles + +- Follow the principle of least privilege +- Never commit secrets or sensitive data to version control +- Use environment variables for all configuration +- Implement defense in depth with multiple security layers +- Regularly update dependencies and base images +- Log security events for monitoring and auditing + +## Authentication and Authorization + +- Implement proper user authentication mechanisms +- Use secure session management practices +- Implement role-based access control where needed +- Validate user permissions for all operations +- Use secure password policies and storage +- Implement account lockout mechanisms + +## Input Validation and Sanitization + +- Validate all user inputs on both client and server side +- Use Pydantic models for API input validation +- Sanitize file paths to prevent directory traversal +- Validate file types and sizes before processing +- Escape output to prevent injection attacks +- Use parameterized queries for database operations + +## Container Security + +- Run containers as non-root users +- Use minimal base images to reduce attack surface +- Scan container images for vulnerabilities regularly +- Keep base images and dependencies updated +- Use read-only filesystems where possible +- Implement proper secrets management + +## Network Security + +- Use HTTPS/TLS for all external communications +- Implement proper CORS policies +- Use secure network configurations in Docker +- Restrict network access to necessary services only +- Monitor network traffic for anomalies +- Use secure protocols for remote storage connections + +## File System Security + +- Validate and sanitize file paths +- Implement proper file permissions +- Use secure temporary file handling +- Prevent directory traversal attacks +- Validate file types and content +- Implement secure file upload mechanisms + +## Data Protection + +- Encrypt sensitive data at rest and in transit +- Implement secure backup and recovery procedures +- Use secure deletion for temporary files +- Implement data retention and disposal policies +- Protect against data leakage in logs and error messages +- Use encryption for remote storage connections + +## API Security + +- Implement rate limiting to prevent abuse +- Use proper HTTP methods and status codes +- Validate all API inputs and parameters +- Implement request size limits +- Use secure headers (CSRF, HSTS, etc.) +- Log and monitor API usage patterns + +## Docker-Specific Security + +- Secure Docker socket access carefully +- Validate Docker API operations +- Implement proper cleanup of Docker resources +- Monitor Docker daemon security events +- Use Docker secrets for sensitive configuration +- Implement resource limits to prevent abuse + +## Frontend Security + +- Implement Content Security Policy (CSP) +- Sanitize user inputs before rendering +- Use secure coding practices for React components +- Implement proper error boundaries +- Avoid storing sensitive data in browser storage +- Use secure communication protocols + +## Logging and Monitoring + +- Log all security-relevant events +- Monitor for unusual access patterns +- Implement alerting for security incidents +- Ensure logs don't contain sensitive information +- Use structured logging for security events +- Implement log retention and archival policies + +## Dependency Security + +- Regularly audit and update dependencies +- Use dependency scanning tools +- Pin dependency versions for reproducibility +- Monitor security advisories for used packages +- Implement automated security updates where appropriate +- Remove unused dependencies regularly + +## Backup Security + +- Encrypt backup data before transmission +- Use secure authentication for remote storage +- Implement integrity checks for backup data +- Secure backup scheduling and automation +- Protect backup metadata and configuration +- Implement secure backup restoration procedures + +## Incident Response + +- Document security incident response procedures +- Implement security event monitoring and alerting +- Prepare rollback and recovery procedures +- Document security contacts and escalation paths +- Regularly test incident response procedures +- Maintain security incident logs and documentation + +## Development Security + +- Use secure development practices +- Implement security code reviews +- Use static analysis security testing (SAST) +- Implement dynamic application security testing (DAST) +- Secure development environment configurations +- Train developers on security best practices \ No newline at end of file diff --git a/.github/instructions/testing.instructions.md b/.github/instructions/testing.instructions.md new file mode 100644 index 0000000..126620e --- /dev/null +++ b/.github/instructions/testing.instructions.md @@ -0,0 +1,119 @@ +--- +applyTo: "**/test_*.py,**/tests/**/*.py,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx" +description: "Testing standards and practices for DockerVault" +--- + +# Testing Guidelines + +## General Testing Principles + +- Maintain test coverage above 80% for both frontend and backend +- Write tests that verify behavior, not implementation details +- Use descriptive test names that explain what is being tested +- Follow the AAA pattern: Arrange, Act, Assert +- Keep tests isolated and independent + +## Backend Testing (Python) + +- Use pytest for all Python testing +- Write unit tests for business logic and utilities +- Create integration tests for API endpoints +- Use pytest fixtures for database and Docker test setup +- Mock external dependencies (Docker daemon, remote storage, file system) +- Test async code properly with pytest-asyncio +- Include performance tests for file operations + +### Database Testing + +- Use in-memory SQLite for faster test execution +- Create fresh database instances for each test +- Use database transactions that can be rolled back +- Test database migrations and schema changes +- Verify proper error handling for database failures + +### API Testing + +- Test all HTTP status codes and response formats +- Verify input validation and error messages +- Test authentication and authorization scenarios +- Include tests for edge cases and boundary conditions +- Use TestClient from FastAPI for endpoint testing + +### Docker Integration Testing + +- Mock Docker SDK calls for unit tests +- Use real Docker daemon for integration tests where necessary +- Test container lifecycle management +- Verify proper cleanup of test containers and volumes +- Test error scenarios like Docker daemon unavailability + +## Frontend Testing (React/TypeScript) + +- Use React Testing Library for component testing +- Write tests that verify user interactions and behavior +- Mock API calls with MSW (Mock Service Worker) +- Test accessibility features and keyboard navigation +- Use Jest for test running and assertions + +### Component Testing + +- Test component rendering with different props +- Verify event handling and state changes +- Test conditional rendering and error states +- Include tests for loading and empty states +- Test responsive behavior where applicable + +### Integration Testing + +- Test complete user workflows +- Verify data flow between components +- Test real-time updates via WebSocket +- Include tests for error boundaries +- Test routing and navigation + +### State Management Testing + +- Test React Query cache behavior and invalidation +- Verify Zustand store updates and selectors +- Test optimistic updates and error handling +- Include tests for offline scenarios + +## End-to-End Testing + +- Test critical backup workflows from UI to completion +- Verify file system operations and backup integrity +- Test user authentication and session management +- Include tests for error recovery and retry mechanisms +- Test backup scheduling and automated operations + +## Test Data Management + +- Use factories or builders for test data creation +- Keep test data minimal and focused +- Use realistic but anonymized test data +- Clean up test files and containers after tests +- Avoid using production data in tests + +## Performance Testing + +- Include load tests for backup operations +- Test memory usage during large file operations +- Verify proper resource cleanup after operations +- Test concurrent backup scenarios +- Monitor test execution time and optimize slow tests + +## Security Testing + +- Test input validation and sanitization +- Verify authentication and authorization controls +- Test for common security vulnerabilities +- Include tests for file path traversal prevention +- Test error messages don't leak sensitive information + +## Test Environment Setup + +- Use Docker containers for consistent test environments +- Include database seeding scripts for integration tests +- Set up CI/CD pipeline to run all tests automatically +- Use environment variables for test configuration +- Implement parallel test execution where possible \ No newline at end of file diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml new file mode 100644 index 0000000..7abcd03 --- /dev/null +++ b/.github/workflows/copilot-setup-steps.yml @@ -0,0 +1,94 @@ +name: "Copilot Setup Steps" +on: + workflow_dispatch: + push: + paths: + - .github/workflows/copilot-setup-steps.yml + pull_request: + paths: + - .github/workflows/copilot-setup-steps.yml + +jobs: + # The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot. + copilot-setup-steps: + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v5 + + # Backend Python setup + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.14" + cache: "pip" + + - name: Install backend dependencies + run: | + cd backend + pip install -r requirements.txt + + - name: Run Python linting + run: | + cd backend + pip install flake8 black isort mypy + flake8 app/ --max-line-length=88 --extend-ignore=E203,W503 + black --check app/ + isort --check-only app/ + mypy app/ + + - name: Run backend tests + run: | + cd backend + pip install pytest pytest-asyncio pytest-cov + pytest tests/ --cov=app --cov-report=term-missing + + # Frontend Node.js setup + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: frontend/package-lock.json + + - name: Install frontend dependencies + run: | + cd frontend + npm ci + + - name: Run frontend linting + run: | + cd frontend + npm run lint + + - name: Run frontend tests + run: | + cd frontend + npm test -- --coverage --watchAll=false + + - name: Build frontend + run: | + cd frontend + npm run build + + # Docker setup and validation + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Validate Docker Compose configuration + run: | + docker-compose config + + - name: Build Docker images + run: | + docker-compose build + + - name: Run Docker Compose health check + run: | + docker-compose up -d + sleep 30 + docker-compose ps + docker-compose down \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9fcc9b4..fcb2447 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,6 +18,7 @@ jobs: outputs: version: ${{ steps.version.outputs.version }} is_prerelease: ${{ steps.version.outputs.is_prerelease }} + build_env: ${{ steps.version.outputs.build_env }} docker_tags: ${{ steps.docker_meta.outputs.tags }} docker_labels: ${{ steps.docker_meta.outputs.labels }} steps: @@ -36,22 +37,27 @@ jobs: # Dev tag (e.g., vdev.0.0.103 -> dev.0.0.103) VERSION=${GITHUB_REF#refs/tags/v} IS_PRERELEASE=true + BUILD_ENV=development elif [[ "${{ github.ref }}" == refs/tags/v* ]]; then # Stable tag (e.g., v1.2.3 -> 1.2.3) VERSION=${GITHUB_REF#refs/tags/v} IS_PRERELEASE=false + BUILD_ENV=production elif [[ "${{ github.ref }}" == refs/heads/main ]]; then # Main branch - stable release with same count as dev VERSION="0.0.${COMMIT_COUNT}" IS_PRERELEASE=false + BUILD_ENV=production else # Develop branch - prerelease with dev prefix VERSION="dev.0.0.${COMMIT_COUNT}" IS_PRERELEASE=true + BUILD_ENV=development fi echo "version=${VERSION}" >> $GITHUB_OUTPUT echo "is_prerelease=${IS_PRERELEASE}" >> $GITHUB_OUTPUT - echo "Version: ${VERSION}, Prerelease: ${IS_PRERELEASE}" + echo "build_env=${BUILD_ENV}" >> $GITHUB_OUTPUT + echo "Version: ${VERSION}, Prerelease: ${IS_PRERELEASE}, Build Env: ${BUILD_ENV}" - name: Docker metadata id: docker_meta @@ -110,6 +116,7 @@ jobs: VERSION=${{ needs.prepare.outputs.version }} COMMIT_SHA=${{ github.sha }} BRANCH=${{ github.ref_name }} + BUILD_ENV=${{ needs.prepare.outputs.build_env }} create-release: needs: [prepare, build-and-push] @@ -162,12 +169,15 @@ jobs: env: CHANGELOG_CONTENT: ${{ steps.changelog.outputs.content }} VERSION: ${{ needs.prepare.outputs.version }} + BUILD_ENV: ${{ needs.prepare.outputs.build_env }} REPO_OWNER: ${{ github.repository_owner }} REPO: ${{ github.repository }} run: | cat > release_notes.md << EOF ## What's Changed in ${VERSION} + **Build Environment:** \`${BUILD_ENV}\` + EOF if [ -n "$CHANGELOG_CONTENT" ]; then @@ -212,6 +222,7 @@ jobs: | Registry | GitHub Container Registry (ghcr.io) | | Image | \`ghcr.io/${REPO_OWNER}/dockervault\` | | Version | \`${VERSION}\` | + | Environment | \`${BUILD_ENV}\` | | Architectures | linux/amd64, linux/arm64 | ## Documentation diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..c606493 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,208 @@ +name: Tests + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + +jobs: + backend-tests: + runs-on: ubuntu-latest + + services: + docker: + image: docker:dind + options: --privileged + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.14 + uses: actions/setup-python@v5 + with: + python-version: '3.14' + + - name: Cache pip dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('backend/requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install dependencies + working-directory: backend + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Run backend tests with coverage + working-directory: backend + run: | + pytest --cov=app --cov-report=xml --cov-report=html --cov-fail-under=80 + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + files: ./backend/coverage.xml + flags: backend + name: backend-coverage + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Archive coverage report + uses: actions/upload-artifact@v4 + with: + name: backend-coverage-report + path: backend/htmlcov/ + + frontend-tests: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js 24 + uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'npm' + cache-dependency-path: frontend/package-lock.json + + - name: Install dependencies + working-directory: frontend + run: npm ci + + - name: Run frontend tests with coverage + working-directory: frontend + run: npm run test:coverage -- --run + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + files: ./frontend/coverage/coverage-final.json + flags: frontend + name: frontend-coverage + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Archive coverage report + uses: actions/upload-artifact@v4 + with: + name: frontend-coverage-report + path: frontend/coverage/ + + lint-and-typecheck: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.14 + uses: actions/setup-python@v5 + with: + python-version: '3.14' + + - name: Set up Node.js 24 + uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'npm' + cache-dependency-path: frontend/package-lock.json + + - name: Install Python dependencies + working-directory: backend + run: | + python -m pip install --upgrade pip + pip install ruff mypy + pip install -r requirements.txt + + - name: Install Node dependencies + working-directory: frontend + run: npm ci + + - name: Lint Python code + working-directory: backend + run: ruff check app/ + + - name: Type check Python code + working-directory: backend + run: mypy app/ + + - name: Lint TypeScript code + working-directory: frontend + run: npm run lint + + - name: Type check TypeScript code + working-directory: frontend + run: npx tsc --noEmit + + integration-tests: + runs-on: ubuntu-latest + needs: [backend-tests, frontend-tests] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Docker images + run: | + docker compose build + + - name: Run integration tests + run: | + docker compose up -d + sleep 30 # Wait for services to start + + # Test API health endpoint + curl -f http://localhost:8000/api/v1/health || exit 1 + + # Test frontend serves correctly + curl -f http://localhost:3000 || exit 1 + + docker compose down + + - name: Clean up + if: always() + run: | + docker compose down -v + docker system prune -af + + security-scan: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.14 + uses: actions/setup-python@v5 + with: + python-version: '3.14' + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@0.28.0 + with: + scan-type: 'fs' + scan-ref: '.' + format: 'sarif' + output: 'trivy-results.sarif' + + - name: Upload Trivy scan results + uses: github/codeql-action/upload-sarif@v3 + if: always() + with: + sarif_file: 'trivy-results.sarif' + + - name: Run pip-audit on Python dependencies + working-directory: backend + run: | + pip install pip-audit + pip-audit -r requirements.txt + + - name: Run npm audit on Node dependencies + working-directory: frontend + run: | + npm audit --audit-level high + diff --git a/.gitignore b/.gitignore index aa1fd9a..d717c08 100644 --- a/.gitignore +++ b/.gitignore @@ -78,5 +78,4 @@ backups/ # git .github/agents -.github/instructions .github/prompts diff --git a/Dockerfile b/Dockerfile index 699a2fc..21d9c35 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,91 +1,139 @@ -# DockerVault - Combined Dockerfile -# Multi-stage build for both backend and frontend +# DockerVault - Optimized Multi-Stage Dockerfile +# Following best practices for smaller, more secure container images # ============================================================================= -# Stage 1: Build Frontend +# Stage 1: Frontend Dependencies # ============================================================================= -FROM node:24-alpine AS frontend-builder +FROM node:24-alpine AS frontend-deps -WORKDIR /app/frontend +WORKDIR /app -# Copy package files +# Copy only package files for dependency caching COPY frontend/package.json frontend/package-lock.json* ./ -# Install dependencies -RUN npm ci || npm install +# Install dependencies (cached if package files unchanged) +RUN npm ci --prefer-offline --no-audit + +# ============================================================================= +# Stage 2: Build Frontend +# ============================================================================= +FROM frontend-deps AS frontend-builder + +# Build argument to control environment +ARG BUILD_ENV=production + +WORKDIR /app # Copy source code COPY frontend/ . -# Build the application +# Set NODE_ENV based on build argument +ENV NODE_ENV=${BUILD_ENV} RUN npm run build # ============================================================================= -# Stage 2: Production Image +# Stage 3: Python Dependencies Builder +# ============================================================================= +FROM python:3.14-slim AS python-deps + +WORKDIR /app + +# Install build dependencies for Python packages +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + libffi-dev \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements first for caching +COPY backend/requirements.txt . + +# Install Python dependencies to a virtual environment +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt + +# ============================================================================= +# Stage 4: Production Image # ============================================================================= -FROM python:3.14-slim +FROM python:3.14-slim AS production # Build arguments ARG VERSION=dev ARG COMMIT_SHA=unknown ARG BRANCH=unknown +ARG BUILD_ENV=production -# Labels -LABEL org.opencontainers.image.title="DockerVault" -LABEL org.opencontainers.image.description="Docker Volume Backup Manager with Web UI" -LABEL org.opencontainers.image.version="${VERSION}" -LABEL org.opencontainers.image.revision="${COMMIT_SHA}" -LABEL org.opencontainers.image.source="https://github.com/Serph91P/DockerVault" +# Labels (OCI standard) +LABEL org.opencontainers.image.title="DockerVault" \ + org.opencontainers.image.description="Docker Volume Backup Manager with Web UI" \ + org.opencontainers.image.version="${VERSION}" \ + org.opencontainers.image.revision="${COMMIT_SHA}" \ + org.opencontainers.image.source="https://github.com/Serph91P/DockerVault" \ + org.opencontainers.image.licenses="MIT" WORKDIR /app -# Install runtime dependencies +# Install only runtime dependencies (no build tools) RUN apt-get update && apt-get install -y --no-install-recommends \ nginx \ supervisor \ curl \ rsync \ openssh-client \ - && rm -rf /var/lib/apt/lists/* + tini \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean -# Copy and install Python requirements -COPY backend/requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +# Copy Python virtual environment from builder +COPY --from=python-deps /opt/venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 -# Copy backend application -COPY backend/app ./app +# Create non-root user before copying files +RUN groupadd --gid 1000 dockervault && \ + useradd --uid 1000 --gid dockervault --shell /bin/bash --create-home dockervault -# Copy frontend build -COPY --from=frontend-builder /app/frontend/dist /usr/share/nginx/html +# Create necessary directories with proper ownership +RUN mkdir -p /app/data /backups /var/log/supervisor /run/nginx && \ + chown -R dockervault:dockervault /app /backups /var/log/supervisor -# Copy nginx configuration -COPY frontend/nginx.conf /etc/nginx/conf.d/default.conf +# Copy backend application +COPY --chown=dockervault:dockervault backend/app ./app -# Create directories and user -RUN useradd -m -u 1000 dockervault && \ - mkdir -p /app/data /backups /var/log/supervisor && \ - chown -R dockervault:dockervault /app /backups +# Copy frontend build from builder stage +COPY --from=frontend-builder /app/dist /usr/share/nginx/html -# Copy supervisor configuration +# Copy configuration files +COPY frontend/nginx.conf /etc/nginx/conf.d/default.conf COPY docker/supervisord.conf /etc/supervisor/conf.d/dockervault.conf +# Fix nginx permissions for non-root operation +RUN chown -R dockervault:dockervault /var/log/nginx /var/lib/nginx /run/nginx && \ + chmod 755 /var/log/nginx /var/lib/nginx /run/nginx + # Environment variables ENV DATABASE_URL=sqlite+aiosqlite:///./data/backup.db \ DOCKER_SOCKET=/var/run/docker.sock \ BACKUP_BASE_PATH=/backups \ TZ=UTC \ VERSION=${VERSION} \ - COMMIT_SHA=${COMMIT_SHA} + COMMIT_SHA=${COMMIT_SHA} \ + APP_ENV=${BUILD_ENV} -# Expose ports +# Expose port EXPOSE 80 # Health check -HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \ - CMD curl -f http://localhost:8000/api/v1/docker/health || exit 1 +HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \ + CMD curl -sf http://localhost:8000/health || exit 1 -# Volumes +# Declare volumes VOLUME ["/app/data", "/backups"] +# Use tini as init system for proper signal handling +ENTRYPOINT ["/usr/bin/tini", "--"] + # Start supervisor CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/README.md b/README.md index ca31b95..12fa701 100644 --- a/README.md +++ b/README.md @@ -1,190 +1,134 @@ -# DockerVault - -A modern, containerized backup system for Docker volumes and host paths with a web interface. - -## Features - -- **Docker Integration**: Automatic detection of containers, volumes, and Compose stacks -- **Flexible Backup Targets**: Containers, volumes, host paths, or entire stacks -- **Dependency Management**: Respects `depends_on` relationships when stopping/starting containers -- **Scheduling**: Cron-based automatic backups with duration estimation -- **GFS Retention**: Grandfather-Father-Son retention strategy per backup target -- **Remote Storage**: Off-site backups via SSH, S3, WebDAV, FTP, or Rclone -- **Komodo Integration**: Optional integration with Komodo for container orchestration -- **Real-time Updates**: WebSocket-based live updates in the frontend -- **Security**: Docker socket mounted read-only - -## Requirements - -- Docker 20.10+ -- Docker Compose 2.0+ -- Linux Host (for Docker socket access) + +
-## Installation +DockerVault logo -### 1. Clone the repository - -```bash -git clone https://github.com/Serph91P/DockerVault.git -cd DockerVault -``` +# DockerVault -### 2. Configure environment variables +**Automated Docker backup solution with a modern web interface** -```bash -cp .env.example .env -``` +[![Docker](https://img.shields.io/badge/Docker-20.10+-2496ED?style=flat-square&logo=docker&logoColor=white)](https://www.docker.com/) +[![Python](https://img.shields.io/badge/Python-3.14-3776AB?style=flat-square&logo=python&logoColor=white)](https://www.python.org/) +[![React](https://img.shields.io/badge/React-18-61DAFB?style=flat-square&logo=react&logoColor=black)](https://react.dev/) +[![FastAPI](https://img.shields.io/badge/FastAPI-009688?style=flat-square&logo=fastapi&logoColor=white)](https://fastapi.tiangolo.com/) +[![License](https://img.shields.io/badge/License-MIT-yellow?style=flat-square)](LICENSE) -Important settings in `.env`: +[Features](#features) โ€ข [Getting Started](#getting-started) โ€ข [Configuration](#configuration) โ€ข [Development](#development) -```env -# Get Docker group ID -DOCKER_GID=$(getent group docker | cut -d: -f3) +
-# Backup storage location -BACKUP_PATH=/path/to/backups +DockerVault is a containerized backup system for Docker volumes and host paths. It provides automatic detection of containers, volumes, and Compose stacks, with flexible scheduling, GFS retention policies, and remote storage synchronization. -# Web interface port -PORT=8080 -``` +## Features -### 3. Start +- **Docker Integration** โ€” Automatic detection of containers, volumes, and Compose stacks +- **Flexible Targets** โ€” Back up containers, volumes, host paths, or entire stacks +- **Dependency Management** โ€” Respects `depends_on` relationships when stopping/starting containers +- **Cron Scheduling** โ€” Automated backups with duration estimation +- **GFS Retention** โ€” Grandfather-Father-Son retention strategy per backup target +- **Remote Storage** โ€” Sync to SSH, S3, WebDAV, FTP, or 40+ providers via Rclone +- **Real-time UI** โ€” WebSocket-based live updates in the web interface +- **Komodo Integration** โ€” Optional integration with Komodo for container orchestration +- **Security First** โ€” Docker socket and volumes mounted read-only -```bash -docker compose up -d -``` +## Getting Started -The web interface is available at `http://localhost:8080`. +### Prerequisites -## Usage +- Docker 20.10+ +- Docker Compose 2.0+ +- Linux host (for Docker socket access) -### Dashboard +### Quick Start -Overview showing: -- Active containers and volumes -- Recent backups with status -- Upcoming scheduled backups -- Statistics +1. **Clone the repository** -### Containers + ```bash + git clone https://github.com/Serph91P/DockerVault.git + cd DockerVault + ``` -- List of all Docker containers -- Status (running/stopped) -- Associated volumes -- Compose stack information -- One-click backup target creation +2. **Configure environment** -### Volumes + ```bash + cp .env.example .env + ``` -- List of all Docker volumes -- Containers using the volume -- Mountpoints -- One-click backup target creation + Edit `.env` with your settings: -### Stacks + ```env + # Docker group ID (find with: getent group docker | cut -d: -f3) + DOCKER_GID=999 -- Docker Compose stacks -- Included containers and volumes -- Network information -- Complete stack as backup target + # Backup storage location + BACKUP_PATH=/path/to/backups -### Backup Targets + # Web interface port + PORT=8080 + ``` -Configured backup targets with: -- Target type (container/volume/path/stack) -- Schedule (cron expression) -- Dependencies -- Pre/Post backup commands -- Container stop/start option -- Compression -- **Individual retention policy** +3. **Start DockerVault** -### Backups + ```bash + docker compose up -d + ``` -- List of all backups -- Status and progress -- File size and duration -- Restore -- Delete +4. **Access the web interface** at `http://localhost:8080` -### Schedules +> [!TIP] +> Use `docker compose logs -f` to monitor startup and check for any configuration issues. -- Overview of scheduled backups -- Cron expression editor -- Next/Last execution -- Manual trigger +## Configuration -### Retention Policies +### Retention Policy -Each backup target can have its own GFS (Grandfather-Father-Son) retention policy: +DockerVault uses a GFS (Grandfather-Father-Son) retention strategy. Each backup target can have its own policy: -| Option | Description | Example | +| Option | Description | Default | |--------|-------------|---------| -| `keep_last` | Keep the last N backups regardless of age | `3` | +| `keep_last` | Keep the last N backups | `3` | | `keep_daily` | Keep one backup per day for N days | `7` | | `keep_weekly` | Keep one backup per week for N weeks | `4` | | `keep_monthly` | Keep one backup per month for N months | `6` | | `keep_yearly` | Keep one backup per year for N years | `2` | -Example configuration (similar to restic): -``` ---keep-last 3 --keep-daily 7 --keep-weekly 4 --keep-monthly 6 --keep-yearly 2 -``` +Configure defaults via environment variables: -This keeps: -- The 3 most recent backups -- 7 daily backups (last week) -- 4 weekly backups (last month) -- 6 monthly backups (last 6 months) -- 2 yearly backups +```env +DEFAULT_KEEP_LAST=3 +DEFAULT_KEEP_DAILY=7 +DEFAULT_KEEP_WEEKLY=4 +DEFAULT_KEEP_MONTHLY=6 +DEFAULT_KEEP_YEARLY=2 +``` ### Remote Storage -Off-site backup synchronization to external storage: +Sync backups to external storage providers: | Type | Description | Example | |------|-------------|---------| -| **Local/NFS** | Local directory or NFS mount | `/mnt/nas/backups` | -| **SSH/SFTP** | SSH server with rsync | `user@server:/backups` | -| **S3** | AWS S3, MinIO, Backblaze B2 | `s3://bucket/path` | -| **WebDAV** | Nextcloud, ownCloud | `https://cloud.example.com/dav/` | -| **FTP/FTPS** | FTP server | `ftp://server/path` | -| **Rclone** | 40+ providers (GDrive, Dropbox, OneDrive, ...) | `remote:path` | - -**Features:** -- Automatic sync after backup -- Configurable per backup target -- Multiple remote destinations -- Connection test in UI -- Encrypted password storage - -## Configuration +| Local/NFS | Local directory or NFS mount | `/mnt/nas/backups` | +| SSH/SFTP | SSH server with rsync | `user@server:/backups` | +| S3 | AWS S3, MinIO, Backblaze B2 | `s3://bucket/path` | +| WebDAV | Nextcloud, ownCloud | `https://cloud.example.com/dav/` | +| FTP/FTPS | FTP server | `ftp://server/path` | +| Rclone | 40+ providers (GDrive, Dropbox, OneDrive, ...) | `remote:path` | ### Cron Expressions -Format: `Minute Hour Day Month Weekday` +Schedule format: `Minute Hour Day Month Weekday` -Examples: -- `0 2 * * *` - Daily at 02:00 -- `0 3 * * 0` - Sundays at 03:00 -- `0 */6 * * *` - Every 6 hours -- `30 1 1 * *` - 1st of every month at 01:30 - -### Default Retention Policy - -Default GFS retention (can be overridden per target): - -```env -DEFAULT_KEEP_LAST=3 -DEFAULT_KEEP_DAILY=7 -DEFAULT_KEEP_WEEKLY=4 -DEFAULT_KEEP_MONTHLY=6 -DEFAULT_KEEP_YEARLY=2 -``` +| Expression | Description | +|------------|-------------| +| `0 2 * * *` | Daily at 02:00 | +| `0 3 * * 0` | Sundays at 03:00 | +| `0 */6 * * *` | Every 6 hours | +| `30 1 1 * *` | 1st of every month at 01:30 | ### Komodo Integration -For integration with Komodo: +Enable optional integration with [Komodo](https://github.com/mbecker20/komodo): ```env KOMODO_ENABLED=true @@ -192,72 +136,17 @@ KOMODO_API_URL=http://komodo:8080 KOMODO_API_KEY=your-api-key ``` -Features: -- Backup notifications -- Container start/stop via Komodo -- Status synchronization - ## Security -### Docker Socket - -The Docker socket is mounted **read-only**: -```yaml -volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro -``` - -### Container Permissions - -The container does not run as root but requires Docker group access: -```yaml -group_add: - - ${DOCKER_GID:-999} -``` - -### Volume Access +DockerVault follows security best practices: -Docker volumes are also mounted read-only: -```yaml -volumes: - - /var/lib/docker/volumes:/var/lib/docker/volumes:ro -``` - -## Project Structure - -``` -DockerVault/ -โ”œโ”€โ”€ backend/ -โ”‚ โ”œโ”€โ”€ app/ -โ”‚ โ”‚ โ”œโ”€โ”€ api/ # REST API endpoints -โ”‚ โ”‚ โ”œโ”€โ”€ main.py # FastAPI application -โ”‚ โ”‚ โ”œโ”€โ”€ config.py # Configuration -โ”‚ โ”‚ โ”œโ”€โ”€ database.py # SQLAlchemy models -โ”‚ โ”‚ โ”œโ”€โ”€ docker_client.py # Docker SDK wrapper -โ”‚ โ”‚ โ”œโ”€โ”€ backup_engine.py # Backup logic -โ”‚ โ”‚ โ”œโ”€โ”€ retention.py # Retention manager -โ”‚ โ”‚ โ”œโ”€โ”€ scheduler.py # APScheduler -โ”‚ โ”‚ โ”œโ”€โ”€ komodo.py # Komodo client -โ”‚ โ”‚ โ”œโ”€โ”€ remote_storage.py # Remote storage backends -โ”‚ โ”‚ โ””โ”€โ”€ websocket.py # WebSocket handler -โ”‚ โ”œโ”€โ”€ Dockerfile -โ”‚ โ””โ”€โ”€ requirements.txt -โ”œโ”€โ”€ frontend/ -โ”‚ โ”œโ”€โ”€ src/ -โ”‚ โ”‚ โ”œโ”€โ”€ api/ # API client -โ”‚ โ”‚ โ”œโ”€โ”€ components/ # React components -โ”‚ โ”‚ โ”œโ”€โ”€ pages/ # Pages -โ”‚ โ”‚ โ””โ”€โ”€ store/ # State (WebSocket) -โ”‚ โ”œโ”€โ”€ Dockerfile -โ”‚ โ””โ”€โ”€ package.json -โ”œโ”€โ”€ docker-compose.yml -โ”œโ”€โ”€ .env.example -โ””โ”€โ”€ README.md -``` +- **Docker socket** โ€” Mounted read-only (`/var/run/docker.sock:ro`) +- **Docker volumes** โ€” Mounted read-only (`/var/lib/docker/volumes:ro`) +- **Non-root user** โ€” Container runs as unprivileged user with Docker group access ## Development -### Start backend locally +### Backend ```bash cd backend @@ -267,7 +156,7 @@ pip install -r requirements.txt uvicorn app.main:app --reload ``` -### Start frontend locally +### Frontend ```bash cd frontend @@ -275,20 +164,38 @@ npm install npm run dev ``` -## API Documentation +### API Documentation -After starting, API documentation is available at: -- Swagger UI: `http://localhost:8000/docs` -- ReDoc: `http://localhost:8000/redoc` +When running, API documentation is available at: +- **Swagger UI**: `http://localhost:8000/docs` +- **ReDoc**: `http://localhost:8000/redoc` -## License +## Project Structure -MIT License +``` +DockerVault/ +โ”œโ”€โ”€ backend/ +โ”‚ โ””โ”€โ”€ app/ +โ”‚ โ”œโ”€โ”€ api/ # REST API endpoints +โ”‚ โ”œโ”€โ”€ backup_engine.py # Backup logic +โ”‚ โ”œโ”€โ”€ docker_client.py # Docker SDK wrapper +โ”‚ โ”œโ”€โ”€ remote_storage.py # Remote storage backends +โ”‚ โ”œโ”€โ”€ retention.py # Retention manager +โ”‚ โ”œโ”€โ”€ scheduler.py # APScheduler integration +โ”‚ โ””โ”€โ”€ websocket.py # Real-time updates +โ”œโ”€โ”€ frontend/ +โ”‚ โ””โ”€โ”€ src/ +โ”‚ โ”œโ”€โ”€ components/ # React components +โ”‚ โ”œโ”€โ”€ pages/ # Application pages +โ”‚ โ””โ”€โ”€ api/ # API client +โ”œโ”€โ”€ docker-compose.yml +โ””โ”€โ”€ Dockerfile +``` -## Acknowledgments +## Resources -- [FastAPI](https://fastapi.tiangolo.com/) -- [React](https://react.dev/) -- [Docker SDK for Python](https://docker-py.readthedocs.io/) -- [APScheduler](https://apscheduler.readthedocs.io/) -- [TailwindCSS](https://tailwindcss.com/) +- [FastAPI](https://fastapi.tiangolo.com/) โ€” Backend framework +- [React](https://react.dev/) โ€” Frontend library +- [Docker SDK for Python](https://docker-py.readthedocs.io/) โ€” Docker integration +- [APScheduler](https://apscheduler.readthedocs.io/) โ€” Job scheduling +- [TailwindCSS](https://tailwindcss.com/) โ€” UI styling diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..ec77bcb --- /dev/null +++ b/TESTING.md @@ -0,0 +1,374 @@ +# Testing Guide for DockerVault + +This document provides comprehensive information about testing DockerVault, including setup, running tests, coverage requirements, and best practices. + +## ๐Ÿงช Test Structure + +### Backend Tests (Python) + +``` +backend/ +โ”œโ”€โ”€ tests/ +โ”‚ โ”œโ”€โ”€ conftest.py # Test configuration and fixtures +โ”‚ โ”œโ”€โ”€ test_backup_engine.py # Backup engine functionality +โ”‚ โ”œโ”€โ”€ test_api_backups.py # Backup API endpoints +โ”‚ โ”œโ”€โ”€ test_docker_client.py # Docker integration +โ”‚ โ”œโ”€โ”€ test_database.py # Database models and operations +โ”‚ โ””โ”€โ”€ test_scheduler.py # Backup scheduling +โ”œโ”€โ”€ pytest.ini # Pytest configuration +โ””โ”€โ”€ requirements-dev.txt # Test dependencies +``` + +### Frontend Tests (TypeScript/React) + +``` +frontend/ +โ”œโ”€โ”€ src/ +โ”‚ โ”œโ”€โ”€ pages/__tests__/ # Page component tests +โ”‚ โ”œโ”€โ”€ store/__tests__/ # State management tests +โ”‚ โ”œโ”€โ”€ api/__tests__/ # API layer tests +โ”‚ โ””โ”€โ”€ test/ # Test utilities +โ”‚ โ”œโ”€โ”€ setup.ts # Test setup configuration +โ”‚ โ””โ”€โ”€ mocks/ # Mock Service Worker handlers +โ”œโ”€โ”€ vitest.config.ts # Vitest configuration +โ””โ”€โ”€ package.json # Test scripts and dependencies +``` + +## ๐Ÿš€ Quick Start + +### Run All Tests + +```bash +# Make test script executable (first time only) +chmod +x test.sh + +# Run all tests with coverage +./test.sh + +# Run with options +./test.sh --help # Show help +./test.sh --backend-only # Only backend tests +./test.sh --frontend-only # Only frontend tests +./test.sh --no-coverage # Skip coverage reports +./test.sh --verbose # Verbose output +``` + +### Backend Tests Only + +```bash +cd backend + +# Create virtual environment (first time) +python -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate + +# Install dependencies +pip install -r requirements.txt +pip install -r requirements-dev.txt + +# Run tests +pytest # Basic run +pytest -v # Verbose output +pytest --cov=app # With coverage +pytest --cov=app --cov-report=html # HTML coverage report +pytest -k "test_backup" # Run specific tests +pytest --tb=short # Short traceback format +``` + +### Frontend Tests Only + +```bash +cd frontend + +# Install dependencies (first time) +npm install + +# Run tests +npm test # Interactive mode +npm test -- --run # Run once +npm run test:coverage # With coverage +npm run test:ui # UI mode (browser) +vitest --watch # Watch mode +``` + +## ๐Ÿ“Š Coverage Requirements + +### Minimum Coverage Thresholds + +- **Overall**: 80% minimum +- **Critical paths**: 100% (backup/restore operations) +- **Security functions**: 100% (input validation, path sanitization) +- **API endpoints**: All status codes tested +- **Error handling**: All exception paths covered + +### Coverage Reports + +After running tests with coverage: + +- **Backend**: `backend/htmlcov/index.html` +- **Frontend**: `frontend/coverage/index.html` + +```bash +# Open coverage reports +open backend/htmlcov/index.html # macOS +open frontend/coverage/index.html # macOS + +xdg-open backend/htmlcov/index.html # Linux +xdg-open frontend/coverage/index.html # Linux +``` + +## ๐Ÿงฉ Test Categories + +### Unit Tests + +Test individual functions, classes, and components in isolation. + +```python +# Backend example +@pytest.mark.asyncio +async def test_backup_creation(): + engine = BackupEngine() + backup = await engine.create_backup(target, BackupType.FULL) + assert backup.status == BackupStatus.PENDING +``` + +```typescript +// Frontend example +it('should render backup status correctly', () => { + render() + expect(screen.getByText('completed')).toBeInTheDocument() +}) +``` + +### Integration Tests + +Test API endpoints and component interactions. + +```python +@pytest.mark.asyncio +async def test_create_backup_api(async_client): + response = await async_client.post("/api/v1/backups", json={ + "target_id": 1, + "backup_type": "full" + }) + assert response.status_code == 201 +``` + +### Security Tests + +Test input validation, authentication, and security measures. + +```python +async def test_path_traversal_prevention(async_client): + response = await async_client.post("/api/v1/backups", json={ + "volume_name": "../../../etc/passwd" + }) + assert response.status_code == 400 +``` + +## ๐Ÿ› ๏ธ Test Utilities and Fixtures + +### Backend Fixtures + +```python +@pytest.fixture +async def db_session(): + """Provide clean database session for each test.""" + async with test_db() as session: + yield session + await session.rollback() + +@pytest.fixture +def mock_docker_client(): + """Mock Docker client for testing.""" + with patch('app.docker_client.docker_client') as mock: + mock.volumes.list.return_value = [] + yield mock +``` + +### Frontend Mocks + +```typescript +// MSW handlers for API mocking +export const handlers = [ + http.get('/api/v1/backups', () => { + return HttpResponse.json(mockBackups) + }), + http.post('/api/v1/backups', async ({ request }) => { + const body = await request.json() + return HttpResponse.json(newBackup, { status: 201 }) + }), +] +``` + +## ๐Ÿ› Debugging Tests + +### Backend Debugging + +```bash +# Run specific test with debugging +pytest -v -s test_backup_engine.py::test_run_backup_volume + +# Use pytest-xdist for parallel execution +pytest -n auto + +# Generate JUnit XML for CI +pytest --junitxml=results.xml +``` + +### Frontend Debugging + +```bash +# Run specific test file +vitest src/pages/__tests__/Backups.test.tsx + +# Run tests matching pattern +vitest --run -t "should handle errors" + +# Debug mode with inspect +vitest --inspect-brk +``` + +### Common Issues + +1. **Async/Await Issues**: Ensure all async operations are properly awaited +2. **Mock Cleanup**: Reset mocks between tests using `beforeEach` +3. **Database State**: Use transactions that can be rolled back +4. **WebSocket Mocks**: Properly mock WebSocket connections +5. **File System**: Use temporary directories for file operations + +## ๐Ÿ” Test Best Practices + +### General Guidelines + +- **Test Behavior, Not Implementation**: Focus on what the code does, not how +- **Use Descriptive Names**: Test names should explain what is being tested +- **Follow AAA Pattern**: Arrange, Act, Assert +- **Keep Tests Independent**: Each test should be able to run in isolation +- **Mock External Dependencies**: Don't rely on external services + +### Backend Best Practices + +```python +# โœ… Good +@pytest.mark.asyncio +async def test_backup_fails_when_docker_unavailable(): + # Arrange + mock_docker.side_effect = DockerException("Docker not available") + + # Act + result = await backup_engine.run_backup(backup_id) + + # Assert + assert result is False + assert backup.status == BackupStatus.FAILED + +# โŒ Bad +async def test_backup(): + result = await backup_engine.run_backup(1) + assert result +``` + +### Frontend Best Practices + +```typescript +// โœ… Good +it('should display error message when backup creation fails', async () => { + // Arrange + server.use( + http.post('/api/v1/backups', () => { + return new HttpResponse(null, { status: 500 }) + }) + ) + + // Act + render(, { wrapper: createWrapper() }) + await user.click(screen.getByRole('button', { name: /create/i })) + + // Assert + await waitFor(() => { + expect(screen.getByText(/error creating backup/i)).toBeInTheDocument() + }) +}) + +// โŒ Bad +it('should work', () => { + render() + expect(screen.getByText('form')).toBeInTheDocument() +}) +``` + +## ๐Ÿš€ Continuous Integration + +Tests run automatically on: + +- **Push to main/develop branches** +- **Pull requests** +- **Scheduled runs** (nightly) + +### GitHub Actions Workflow + +```yaml +# .github/workflows/test.yml +jobs: + backend-tests: + runs-on: ubuntu-latest + steps: + - name: Run tests with coverage + run: pytest --cov=app --cov-fail-under=80 + + frontend-tests: + runs-on: ubuntu-latest + steps: + - name: Run tests with coverage + run: npm run test:coverage -- --run +``` + +### Coverage Upload + +Coverage reports are automatically uploaded to Codecov and available as artifacts. + +## ๐Ÿ“‹ Testing Checklist + +Before submitting a PR, ensure: + +- [ ] All tests pass locally +- [ ] Coverage meets 80% minimum threshold +- [ ] New features have corresponding tests +- [ ] Security-sensitive code has 100% coverage +- [ ] API endpoints test all status codes +- [ ] Error handling is thoroughly tested +- [ ] Mock external dependencies appropriately +- [ ] Tests follow naming conventions +- [ ] Test documentation is updated + +## ๐Ÿ›ก๏ธ Security Testing + +Special attention to: + +- **Path Traversal**: Test `../../../etc/passwd` scenarios +- **SQL Injection**: Test malicious input in database queries +- **Command Injection**: Test shell command sanitization +- **Input Validation**: Test boundary conditions and invalid input +- **Authentication**: Test unauthorized access attempts +- **File Operations**: Test secure file handling + +## ๐Ÿ“š Additional Resources + +- [pytest Documentation](https://docs.pytest.org/) +- [Vitest Documentation](https://vitest.dev/) +- [React Testing Library](https://testing-library.com/docs/react-testing-library/intro/) +- [MSW Documentation](https://mswjs.io/docs/) +- [FastAPI Testing](https://fastapi.tiangolo.com/tutorial/testing/) + +## ๐Ÿค Contributing Tests + +When adding new tests: + +1. **Follow existing patterns** in the codebase +2. **Add tests for new features** before implementing +3. **Update documentation** if test structure changes +4. **Ensure tests are deterministic** and don't rely on timing +5. **Use appropriate test categories** (unit, integration, e2e) + +For questions about testing, please refer to the [Contributing Guide](CONTRIBUTING.md) or open an issue. diff --git a/backend/app/api/backups.py b/backend/app/api/backups.py index ea51ab5..4b82ee2 100644 --- a/backend/app/api/backups.py +++ b/backend/app/api/backups.py @@ -65,10 +65,23 @@ async def list_backups( target_id: Optional[int] = None, status: Optional[str] = None, limit: int = 50, + offset: int = 0, ): - """List backups with optional filters.""" + """List backups with optional filters and pagination. + + Args: + target_id: Filter by target ID + status: Filter by backup status + limit: Maximum number of results (default 50) + offset: Number of results to skip for pagination (default 0) + """ async with async_session() as session: - query = select(Backup).order_by(Backup.created_at.desc()).limit(limit) + query = ( + select(Backup) + .order_by(Backup.created_at.desc()) + .limit(limit) + .offset(offset) + ) if target_id: query = query.where(Backup.target_id == target_id) @@ -266,3 +279,73 @@ async def get_backup_stats(backup_id: int): "duration_seconds": backup.duration_seconds, "status": backup.status.value, } + + +@router.get("/metrics/summary") +async def get_backup_metrics(): + """Get overall backup metrics and statistics. + + Returns aggregate statistics about backup operations including: + - Total number of backups + - Success/failure counts and rate + - Total data backed up + - Last backup timestamp + """ + return backup_engine.metrics.to_dict() + + +@router.get("/metrics/target/{target_id}") +async def get_target_metrics(target_id: int): + """Get metrics for a specific backup target. + + Returns: + - Average backup duration + - Average backup size + - Recent backup history + """ + avg_duration = backup_engine.metrics.get_average_duration(target_id) + avg_size = backup_engine.metrics.get_average_size(target_id) + + return { + "target_id": target_id, + "average_duration_seconds": avg_duration, + "average_size_bytes": avg_size, + "average_size_human": format_size(avg_size) if avg_size else None, + "backup_count": len(backup_engine.metrics.target_durations.get(target_id, [])), + } + + +@router.post("/{backup_id}/validate") +async def validate_backup_target(backup_id: int): + """Validate backup prerequisites before running. + + Checks: + - Target container/volume/path exists + - Sufficient disk space + - Dependencies are available + + Returns list of validation issues, or empty list if valid. + """ + async with async_session() as session: + result = await session.execute( + select(Backup).where(Backup.id == backup_id) + ) + backup = result.scalar_one_or_none() + + if not backup: + raise HTTPException(status_code=404, detail="Backup not found") + + result = await session.execute( + select(BackupTarget).where(BackupTarget.id == backup.target_id) + ) + target = result.scalar_one_or_none() + + if not target: + raise HTTPException(status_code=404, detail="Target not found") + + issues = await backup_engine.validate_backup_prerequisites(target) + + return { + "valid": len(issues) == 0, + "issues": issues, + } diff --git a/backend/app/backup_engine.py b/backend/app/backup_engine.py index c1c43b8..3ee12b0 100644 --- a/backend/app/backup_engine.py +++ b/backend/app/backup_engine.py @@ -9,6 +9,9 @@ import gzip import shutil import shlex +import time +from collections import defaultdict +from dataclasses import dataclass, field from datetime import datetime from pathlib import Path from typing import Optional, List, Dict, Any, Callable @@ -22,12 +25,83 @@ logger = logging.getLogger(__name__) +@dataclass +class BackupMetrics: + """Metrics for backup operations.""" + total_backups: int = 0 + successful_backups: int = 0 + failed_backups: int = 0 + total_bytes_backed_up: int = 0 + target_durations: Dict[int, List[float]] = field(default_factory=lambda: defaultdict(list)) + target_sizes: Dict[int, List[int]] = field(default_factory=lambda: defaultdict(list)) + last_backup_time: Optional[datetime] = None + + @property + def success_rate(self) -> float: + """Calculate overall success rate.""" + if self.total_backups == 0: + return 0.0 + return (self.successful_backups / self.total_backups) * 100 + + def record_backup( + self, + target_id: int, + duration: float, + size: int, + success: bool, + ): + """Record metrics for a completed backup.""" + self.total_backups += 1 + if success: + self.successful_backups += 1 + self.total_bytes_backed_up += size + else: + self.failed_backups += 1 + + self.target_durations[target_id].append(duration) + self.target_sizes[target_id].append(size) + self.last_backup_time = datetime.utcnow() + + # Keep only last 100 entries per target to limit memory + if len(self.target_durations[target_id]) > 100: + self.target_durations[target_id] = self.target_durations[target_id][-100:] + if len(self.target_sizes[target_id]) > 100: + self.target_sizes[target_id] = self.target_sizes[target_id][-100:] + + def get_average_duration(self, target_id: int) -> Optional[float]: + """Get average backup duration for a target.""" + durations = self.target_durations.get(target_id, []) + if not durations: + return None + return sum(durations) / len(durations) + + def get_average_size(self, target_id: int) -> Optional[int]: + """Get average backup size for a target.""" + sizes = self.target_sizes.get(target_id, []) + if not sizes: + return None + return int(sum(sizes) / len(sizes)) + + def to_dict(self) -> Dict[str, Any]: + """Convert metrics to dictionary for API response.""" + return { + "total_backups": self.total_backups, + "successful_backups": self.successful_backups, + "failed_backups": self.failed_backups, + "success_rate": round(self.success_rate, 2), + "total_bytes_backed_up": self.total_bytes_backed_up, + "last_backup_time": self.last_backup_time.isoformat() if self.last_backup_time else None, + } + + class BackupEngine: """Handles backup operations.""" def __init__(self): self.active_backups: Dict[int, asyncio.Task] = {} self.progress_callbacks: List[Callable] = [] + self.metrics = BackupMetrics() + self._backup_semaphore = asyncio.Semaphore(settings.MAX_CONCURRENT_BACKUPS) def add_progress_callback(self, callback: Callable): """Add a callback for progress updates.""" @@ -46,6 +120,117 @@ async def _notify_progress(self, backup_id: int, progress: float, message: str): except Exception as e: logger.error(f"Progress callback error: {e}") + async def validate_backup_prerequisites( + self, + target: BackupTarget, + ) -> List[str]: + """Validate backup prerequisites and return any issues. + + Checks: + - Container/volume/path existence + - Available disk space + - Permission issues + + Args: + target: The backup target to validate + + Returns: + List of issue descriptions, empty if validation passes + """ + issues = [] + + if target.target_type == "container" and target.container_name: + containers = await docker_client.list_containers() + if not any(c.name == target.container_name for c in containers): + issues.append(f"Container '{target.container_name}' not found") + logger.warning( + f"Validation failed: container not found", + extra={ + "target_id": target.id, + "target_name": target.name, + "container_name": target.container_name, + } + ) + + elif target.target_type == "volume" and target.volume_name: + volumes = await docker_client.list_volumes() + if not any(v.name == target.volume_name for v in volumes): + issues.append(f"Volume '{target.volume_name}' not found") + logger.warning( + f"Validation failed: volume not found", + extra={ + "target_id": target.id, + "target_name": target.name, + "volume_name": target.volume_name, + } + ) + + elif target.target_type == "path" and target.host_path: + if not os.path.exists(target.host_path): + issues.append(f"Path '{target.host_path}' not found") + logger.warning( + f"Validation failed: path not found", + extra={ + "target_id": target.id, + "target_name": target.name, + "host_path": target.host_path, + } + ) + elif not os.access(target.host_path, os.R_OK): + issues.append(f"Path '{target.host_path}' is not readable") + logger.warning( + f"Validation failed: path not readable", + extra={ + "target_id": target.id, + "target_name": target.name, + "host_path": target.host_path, + } + ) + + # Check available disk space in backup directory + backup_dir = Path(settings.BACKUP_BASE_PATH) + try: + backup_dir.mkdir(parents=True, exist_ok=True) + stat = shutil.disk_usage(backup_dir) + # Warn if less than 1GB free + min_free_bytes = 1024 * 1024 * 1024 # 1 GB + if stat.free < min_free_bytes: + issues.append( + f"Low disk space in backup directory: {stat.free / (1024**3):.2f} GB free" + ) + logger.warning( + f"Low disk space in backup directory", + extra={ + "target_id": target.id, + "free_bytes": stat.free, + "min_required_bytes": min_free_bytes, + } + ) + except Exception as e: + issues.append(f"Cannot check disk space: {e}") + logger.error( + f"Failed to check disk space", + extra={"error": str(e), "error_type": type(e).__name__}, + ) + + # Validate dependencies if specified + if target.dependencies: + containers = await docker_client.list_containers() + container_names = {c.name for c in containers} + for dep in target.dependencies: + if dep not in container_names: + issues.append(f"Dependency container '{dep}' not found") + logger.warning( + f"Validation failed: dependency not found", + extra={ + "target_id": target.id, + "target_name": target.name, + "dependency": dep, + } + ) + + return issues + async def create_backup( self, target: BackupTarget, @@ -57,7 +242,7 @@ async def create_backup( target_id=target.id, backup_type=backup_type, status=BackupStatus.PENDING, - metadata={ + backup_metadata={ "target_name": target.name, "target_type": target.target_type, }, @@ -67,34 +252,86 @@ async def create_backup( await session.refresh(backup) return backup - async def run_backup(self, backup_id: int) -> bool: - """Run a backup by ID.""" - async with async_session() as session: - result = await session.execute( - select(Backup).where(Backup.id == backup_id) - ) - backup = result.scalar_one_or_none() - - if not backup: - logger.error(f"Backup {backup_id} not found") - return False - - # Get target - result = await session.execute( - select(BackupTarget).where(BackupTarget.id == backup.target_id) - ) - target = result.scalar_one_or_none() - - if not target: - logger.error(f"Target for backup {backup_id} not found") - return False + async def run_backup(self, backup_id: int, skip_validation: bool = False) -> bool: + """Run a backup by ID. + + Uses a semaphore to limit concurrent backups based on + MAX_CONCURRENT_BACKUPS setting. + + Args: + backup_id: ID of the backup to run + skip_validation: Skip pre-backup validation (for testing) - # Update status to running - backup.status = BackupStatus.RUNNING - backup.started_at = datetime.utcnow() - await session.commit() + Returns: + True if backup succeeded, False otherwise + """ + start_time = time.time() + file_size = 0 + target_id = None - await self._notify_progress(backup_id, 0, "Starting backup...") + async with self._backup_semaphore: + async with async_session() as session: + result = await session.execute( + select(Backup).where(Backup.id == backup_id) + ) + backup = result.scalar_one_or_none() + + if not backup: + logger.error( + f"Backup {backup_id} not found", + extra={"backup_id": backup_id}, + ) + return False + + # Get target + result = await session.execute( + select(BackupTarget).where(BackupTarget.id == backup.target_id) + ) + target = result.scalar_one_or_none() + + if not target: + logger.error( + f"Target for backup {backup_id} not found", + extra={ + "backup_id": backup_id, + "target_id": backup.target_id, + }, + ) + return False + + target_id = target.id + + # Validate prerequisites unless skipped + if not skip_validation: + validation_issues = await self.validate_backup_prerequisites(target) + if validation_issues: + error_msg = f"Validation failed: {'; '.join(validation_issues)}" + logger.error( + f"Backup {backup_id} validation failed", + extra={ + "backup_id": backup_id, + "target_id": target.id, + "target_name": target.name, + "issues": validation_issues, + }, + ) + backup.status = BackupStatus.FAILED + backup.error_message = error_msg + backup.completed_at = datetime.utcnow() + await session.commit() + await self._notify_progress(backup_id, -1, error_msg) + + # Record failed backup metrics + duration = time.time() - start_time + self.metrics.record_backup(target_id, duration, 0, False) + return False + + # Update status to running + backup.status = BackupStatus.RUNNING + backup.started_at = datetime.utcnow() + await session.commit() + + await self._notify_progress(backup_id, 0, "Starting backup...") try: # Determine containers to stop @@ -157,6 +394,9 @@ async def run_backup(self, backup_id: int) -> bool: # Update backup record async with async_session() as session: + duration_seconds = int( + (datetime.utcnow() - backup.started_at).total_seconds() + ) await session.execute( update(Backup) .where(Backup.id == backup_id) @@ -166,19 +406,40 @@ async def run_backup(self, backup_id: int) -> bool: file_size=file_size, checksum=checksum, completed_at=datetime.utcnow(), - duration_seconds=int( - (datetime.utcnow() - backup.started_at).total_seconds() - ), + duration_seconds=duration_seconds, ) ) await session.commit() + # Record successful backup metrics + duration = time.time() - start_time + self.metrics.record_backup(target_id, duration, file_size, True) + await self._notify_progress(backup_id, 100, "Backup completed!") - logger.info(f"Backup {backup_id} completed successfully") + logger.info( + f"Backup {backup_id} completed successfully", + extra={ + "backup_id": backup_id, + "target_id": target_id, + "target_name": target.name, + "file_size": file_size, + "duration_seconds": duration_seconds, + "backup_path": backup_path, + }, + ) return True except Exception as e: - logger.error(f"Backup {backup_id} failed: {e}") + logger.error( + f"Backup {backup_id} failed: {e}", + extra={ + "backup_id": backup_id, + "target_id": target_id, + "target_name": target.name if target else None, + "error_type": type(e).__name__, + "error": str(e), + }, + ) # Try to restart containers on failure for container_name in reversed(containers_to_stop): @@ -186,7 +447,13 @@ async def run_backup(self, backup_id: int) -> bool: try: await docker_client.start_container(container_name) except Exception as restart_error: - logger.error(f"Failed to restart {container_name}: {restart_error}") + logger.error( + f"Failed to restart {container_name}: {restart_error}", + extra={ + "container_name": container_name, + "error": str(restart_error), + }, + ) # Update backup record with error async with async_session() as session: @@ -201,6 +468,11 @@ async def run_backup(self, backup_id: int) -> bool: ) await session.commit() + # Record failed backup metrics + duration = time.time() - start_time + if target_id: + self.metrics.record_backup(target_id, duration, 0, False) + await self._notify_progress(backup_id, -1, f"Backup failed: {e}") return False diff --git a/backend/app/database.py b/backend/app/database.py index 9fdf9a9..e68a3a1 100644 --- a/backend/app/database.py +++ b/backend/app/database.py @@ -115,8 +115,8 @@ class Backup(Base): error_message = Column(Text, nullable=True) retry_count = Column(Integer, default=0) - # Metadata - metadata = Column(JSON, default=dict) + # Extra data + backup_metadata = Column(JSON, default=dict) created_at = Column(DateTime, default=datetime.utcnow) @@ -233,3 +233,7 @@ async def get_session() -> AsyncSession: """Get database session.""" async with async_session() as session: yield session + + +# Alias for backwards compatibility +get_db = get_session diff --git a/backend/pytest.ini b/backend/pytest.ini new file mode 100644 index 0000000..3d410f3 --- /dev/null +++ b/backend/pytest.ini @@ -0,0 +1,23 @@ +[tool:pytest] +testpaths = tests +python_files = test_*.py +python_functions = test_* +python_classes = Test* +addopts = + --strict-markers + --strict-config + --verbose + --tb=short + --cov=app + --cov-report=term-missing + --cov-report=html:htmlcov + --cov-fail-under=80 +markers = + asyncio: marks tests as async + unit: marks tests as unit tests + integration: marks tests as integration tests + slow: marks tests as slow running +filterwatings = + ignore::DeprecationWarning + ignore::PendingDeprecationWarning +asyncio_mode = auto diff --git a/backend/requirements-dev.txt b/backend/requirements-dev.txt new file mode 100644 index 0000000..0a1800b --- /dev/null +++ b/backend/requirements-dev.txt @@ -0,0 +1,8 @@ +# Testing dependencies +pytest>=7.4.0 +pytest-asyncio>=0.23.0 +pytest-cov>=4.1.0 +pytest-mock>=3.12.0 +httpx>=0.26.0 +factory-boy>=3.3.0 +faker>=22.0.0 diff --git a/backend/tests/__init__.py b/backend/tests/__init__.py new file mode 100644 index 0000000..ff88dbf --- /dev/null +++ b/backend/tests/__init__.py @@ -0,0 +1 @@ +"""Test package for DockerVault backend.""" diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 0000000..9dbca8a --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,124 @@ +""" +Test configuration and shared fixtures. +""" + +import asyncio +import os +import tempfile +from pathlib import Path +from typing import AsyncGenerator, Generator +import pytest +import pytest_asyncio +from httpx import AsyncClient, ASGITransport +from sqlalchemy import create_engine +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker +from unittest.mock import AsyncMock, MagicMock, patch + +from app.main import app +from app.database import get_db, Base +from app.config import settings + +# Override settings for testing +settings.DATABASE_URL = "sqlite+aiosqlite:///:memory:" +settings.BACKUP_BASE_PATH = "/tmp/test_backups" + + +@pytest.fixture(scope="session") +def event_loop(): + """Create event loop for the test session.""" + loop = asyncio.new_event_loop() + yield loop + loop.close() + + +@pytest_asyncio.fixture +async def test_db(): + """Create test database.""" + engine = create_async_engine(settings.DATABASE_URL, echo=False) + + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + async_session_maker = async_sessionmaker( + engine, class_=AsyncSession, expire_on_commit=False + ) + + yield async_session_maker + + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.drop_all) + await engine.dispose() + + +@pytest_asyncio.fixture +async def db_session(test_db): + """Create database session for tests.""" + async with test_db() as session: + yield session + await session.rollback() + + +@pytest_asyncio.fixture +async def async_client(test_db): + """Create async test client.""" + async def override_get_db(): + async with test_db() as session: + yield session + + app.dependency_overrides[get_db] = override_get_db + + async with AsyncClient( + transport=ASGITransport(app=app), + base_url="http://test", + ) as client: + yield client + + app.dependency_overrides.clear() + + +@pytest.fixture +def mock_docker_client(): + """Mock Docker client for testing.""" + with patch('app.docker_client.docker_client') as mock: + # Configure common mock responses + mock.volumes.list.return_value = [] + mock.containers.list.return_value = [] + mock.api.ping.return_value = True + yield mock + + +@pytest.fixture +def temp_backup_dir(): + """Create temporary directory for backup tests.""" + with tempfile.TemporaryDirectory() as temp_dir: + yield Path(temp_dir) + + +@pytest.fixture +def mock_file_operations(): + """Mock file operations for testing.""" + with patch('builtins.open'), \ + patch('os.path.exists') as mock_exists, \ + patch('os.makedirs') as mock_makedirs, \ + patch('shutil.copy2') as mock_copy, \ + patch('tarfile.open') as mock_tarfile: + + mock_exists.return_value = True + yield { + 'exists': mock_exists, + 'makedirs': mock_makedirs, + 'copy': mock_copy, + 'tarfile': mock_tarfile, + } + + +@pytest.fixture +def mock_remote_storage(): + """Mock remote storage operations.""" + with patch('app.remote_storage.RemoteStorage') as mock: + instance = mock.return_value + instance.upload.return_value = AsyncMock(return_value=True) + instance.download.return_value = AsyncMock(return_value=True) + instance.delete.return_value = AsyncMock(return_value=True) + instance.list.return_value = AsyncMock(return_value=[]) + yield instance diff --git a/backend/tests/test_api_backups.py b/backend/tests/test_api_backups.py new file mode 100644 index 0000000..5a07219 --- /dev/null +++ b/backend/tests/test_api_backups.py @@ -0,0 +1,545 @@ +""" +Tests for backups API endpoints. +""" + +import json +from unittest.mock import patch, AsyncMock, MagicMock +import pytest +from httpx import AsyncClient + +from app.database import Backup, BackupTarget, BackupStatus, BackupType + + +@pytest.mark.asyncio +class TestBackupsAPI: + """Test backups API endpoints.""" + + async def test_list_backups_empty(self, async_client: AsyncClient): + """Test listing backups when none exist.""" + response = await async_client.get("/api/v1/backups") + assert response.status_code == 200 + assert response.json() == [] + + async def test_list_backups_with_data(self, async_client: AsyncClient, db_session): + """Test listing backups with data.""" + # Create target and backup + target = BackupTarget( + name="test-target", + target_type="volume", + volume_name="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.COMPLETED, + file_path="/backups/test.tar.gz", + file_size=1024, + checksum="abc123", + backup_metadata={"target_name": "test-target"} + ) + db_session.add(backup) + await db_session.commit() + + response = await async_client.get("/api/v1/backups") + assert response.status_code == 200 + + data = response.json() + assert len(data) == 1 + assert data[0]["id"] == backup.id + assert data[0]["target_id"] == target.id + assert data[0]["target_name"] == "test-target" + assert data[0]["status"] == "completed" + assert data[0]["file_size"] == 1024 + assert data[0]["checksum"] == "abc123" + + @patch('app.api.backups.backup_engine') + async def test_create_backup_success(self, mock_engine, async_client: AsyncClient, db_session): + """Test successful backup creation.""" + # Create target + target = BackupTarget( + name="test-target", + target_type="volume", + volume_name="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + # Mock backup engine + mock_backup = Backup( + id=1, + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.PENDING + ) + mock_engine.create_backup.return_value = mock_backup + mock_engine.run_backup.return_value = AsyncMock(return_value=True) + + response = await async_client.post("/api/v1/backups", json={ + "target_id": target.id, + "backup_type": "full" + }) + + assert response.status_code == 201 + data = response.json() + assert data["target_id"] == target.id + assert data["backup_type"] == "full" + assert data["status"] == "pending" + + async def test_create_backup_invalid_target(self, async_client: AsyncClient): + """Test backup creation with invalid target.""" + response = await async_client.post("/api/v1/backups", json={ + "target_id": 99999, + "backup_type": "full" + }) + + assert response.status_code == 404 + assert "Target not found" in response.json()["detail"] + + async def test_create_backup_invalid_type(self, async_client: AsyncClient, db_session): + """Test backup creation with invalid backup type.""" + # Create target + target = BackupTarget( + name="test-target", + target_type="volume", + volume_name="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + response = await async_client.post("/api/v1/backups", json={ + "target_id": target.id, + "backup_type": "invalid_type" + }) + + assert response.status_code == 422 # Validation error + + async def test_create_backup_disabled_target(self, async_client: AsyncClient, db_session): + """Test backup creation with disabled target.""" + # Create disabled target + target = BackupTarget( + name="disabled-target", + target_type="volume", + volume_name="test-volume", + enabled=False # Disabled + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + response = await async_client.post("/api/v1/backups", json={ + "target_id": target.id, + "backup_type": "full" + }) + + assert response.status_code == 400 + assert "disabled" in response.json()["detail"].lower() + + async def test_get_backup_by_id(self, async_client: AsyncClient, db_session): + """Test getting backup by ID.""" + # Create target and backup + target = BackupTarget( + name="test-target", + target_type="volume", + volume_name="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.COMPLETED, + backup_metadata={"target_name": "test-target"} + ) + db_session.add(backup) + await db_session.commit() + await db_session.refresh(backup) + + response = await async_client.get(f"/api/v1/backups/{backup.id}") + assert response.status_code == 200 + + data = response.json() + assert data["id"] == backup.id + assert data["target_id"] == target.id + assert data["status"] == "completed" + + async def test_get_backup_not_found(self, async_client: AsyncClient): + """Test getting non-existent backup.""" + response = await async_client.get("/api/v1/backups/99999") + assert response.status_code == 404 + assert "Backup not found" in response.json()["detail"] + + @patch('os.path.exists') + @patch('os.remove') + async def test_delete_backup(self, mock_remove, mock_exists, async_client: AsyncClient, db_session): + """Test backup deletion.""" + mock_exists.return_value = True + + # Create target and backup + target = BackupTarget( + name="test-target", + target_type="volume", + volume_name="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.COMPLETED, + file_path="/backups/test.tar.gz", + backup_metadata={"target_name": "test-target"} + ) + db_session.add(backup) + await db_session.commit() + await db_session.refresh(backup) + + response = await async_client.delete(f"/api/v1/backups/{backup.id}") + assert response.status_code == 204 + + # Verify file deletion was attempted + mock_remove.assert_called_once_with("/backups/test.tar.gz") + + # Verify backup was removed from database + response = await async_client.get(f"/api/v1/backups/{backup.id}") + assert response.status_code == 404 + + async def test_security_sql_injection_prevention(self, async_client: AsyncClient): + """Test that SQL injection attempts are blocked.""" + # Try SQL injection in backup ID parameter + malicious_id = "1; DROP TABLE backups; --" + response = await async_client.get(f"/api/v1/backups/{malicious_id}") + + # Should return 422 (validation error) not 500 (server error) + assert response.status_code == 422 + + async def test_input_validation_large_backup_type(self, async_client: AsyncClient, db_session): + """Test input validation for oversized backup type.""" + target = BackupTarget( + name="test-target", + target_type="volume", + volume_name="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + response = await async_client.post("/api/v1/backups", json={ + "target_id": target.id, + "backup_type": "x" * 1000 # Very long string + }) + + assert response.status_code == 422 # Validation error + + @patch('app.api.backups.backup_engine') + async def test_restore_backup_success(self, mock_engine, async_client: AsyncClient, db_session): + """Test successful backup restoration.""" + # Create target and backup + target = BackupTarget( + name="test-target", + target_type="volume", + volume_name="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.COMPLETED, + file_path="/backups/test.tar.gz", + backup_metadata={"target_name": "test-target"} + ) + db_session.add(backup) + await db_session.commit() + await db_session.refresh(backup) + + # Mock restore operation + mock_engine.restore_backup.return_value = AsyncMock(return_value=True) + + response = await async_client.post(f"/api/v1/backups/{backup.id}/restore") + assert response.status_code == 200 + + data = response.json() + assert data["message"] == "Restore initiated" + + # Verify restore was called + mock_engine.restore_backup.assert_called_once_with(backup.id, None) + + async def test_restore_backup_path_traversal_prevention(self, async_client: AsyncClient, db_session): + """Test that restore prevents path traversal attacks.""" + # Create target and backup + target = BackupTarget( + name="test-target", + target_type="volume", + volume_name="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.COMPLETED, + file_path="/backups/test.tar.gz", + backup_metadata={"target_name": "test-target"} + ) + db_session.add(backup) + await db_session.commit() + await db_session.refresh(backup) + + # Attempt path traversal in restore request + response = await async_client.post( + f"/api/v1/backups/{backup.id}/restore", + json={"target_path": "../../../etc/passwd"} + ) + + assert response.status_code == 400 + assert "invalid path" in response.json()["detail"].lower() + + +@pytest.mark.asyncio +class TestBackupsPaginationAPI: + """Test backups API pagination functionality.""" + + async def test_list_backups_with_pagination(self, async_client: AsyncClient, db_session): + """Test listing backups with pagination.""" + # Create target + target = BackupTarget( + name="test-target", + target_type="volume", + volume_name="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + # Create multiple backups + for i in range(15): + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.COMPLETED, + backup_metadata={"target_name": "test-target", "index": i} + ) + db_session.add(backup) + await db_session.commit() + + # Test first page + response = await async_client.get("/api/v1/backups?limit=5&offset=0") + assert response.status_code == 200 + data = response.json() + assert len(data) == 5 + + # Test second page + response = await async_client.get("/api/v1/backups?limit=5&offset=5") + assert response.status_code == 200 + data = response.json() + assert len(data) == 5 + + # Test third page + response = await async_client.get("/api/v1/backups?limit=5&offset=10") + assert response.status_code == 200 + data = response.json() + assert len(data) == 5 + + # Test beyond available data + response = await async_client.get("/api/v1/backups?limit=5&offset=20") + assert response.status_code == 200 + data = response.json() + assert len(data) == 0 + + async def test_list_backups_default_pagination(self, async_client: AsyncClient, db_session): + """Test default pagination values.""" + # Create target + target = BackupTarget( + name="test-target", + target_type="volume", + volume_name="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + # Create 60 backups (more than default limit) + for i in range(60): + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.COMPLETED, + backup_metadata={"target_name": "test-target"} + ) + db_session.add(backup) + await db_session.commit() + + # Without pagination params, should use defaults + response = await async_client.get("/api/v1/backups") + assert response.status_code == 200 + data = response.json() + # Default limit is 50 + assert len(data) == 50 + + +@pytest.mark.asyncio +class TestBackupsMetricsAPI: + """Test backups metrics API endpoints.""" + + @patch('app.api.backups.backup_engine') + async def test_get_backup_metrics_summary(self, mock_engine, async_client: AsyncClient): + """Test getting backup metrics summary.""" + # Mock metrics + mock_engine.metrics.to_dict.return_value = { + "total_backups": 100, + "successful_backups": 95, + "failed_backups": 5, + "success_rate": 95.0, + "total_bytes_backed_up": 1073741824, # 1 GB + "last_backup_time": "2026-01-24T12:00:00", + } + + response = await async_client.get("/api/v1/backups/metrics/summary") + assert response.status_code == 200 + + data = response.json() + assert data["total_backups"] == 100 + assert data["successful_backups"] == 95 + assert data["failed_backups"] == 5 + assert data["success_rate"] == 95.0 + + @patch('app.api.backups.backup_engine') + async def test_get_target_metrics(self, mock_engine, async_client: AsyncClient): + """Test getting metrics for a specific target.""" + # Mock metrics methods + mock_engine.metrics.get_average_duration.return_value = 120.5 + mock_engine.metrics.get_average_size.return_value = 1048576 # 1 MB + mock_engine.metrics.target_durations = {1: [100, 120, 141.5]} + + response = await async_client.get("/api/v1/backups/metrics/target/1") + assert response.status_code == 200 + + data = response.json() + assert data["target_id"] == 1 + assert data["average_duration_seconds"] == 120.5 + assert data["average_size_bytes"] == 1048576 + assert "average_size_human" in data + assert data["backup_count"] == 3 + + @patch('app.api.backups.backup_engine') + async def test_get_target_metrics_no_data(self, mock_engine, async_client: AsyncClient): + """Test getting metrics for target with no data.""" + mock_engine.metrics.get_average_duration.return_value = None + mock_engine.metrics.get_average_size.return_value = None + mock_engine.metrics.target_durations = {} + + response = await async_client.get("/api/v1/backups/metrics/target/999") + assert response.status_code == 200 + + data = response.json() + assert data["target_id"] == 999 + assert data["average_duration_seconds"] is None + assert data["average_size_bytes"] is None + assert data["backup_count"] == 0 + + +@pytest.mark.asyncio +class TestBackupsValidationAPI: + """Test backup validation API endpoints.""" + + @patch('app.api.backups.backup_engine') + async def test_validate_backup_success(self, mock_engine, async_client: AsyncClient, db_session): + """Test validating backup prerequisites successfully.""" + # Create target and backup + target = BackupTarget( + name="test-target", + target_type="volume", + volume_name="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.PENDING, + backup_metadata={"target_name": "test-target"} + ) + db_session.add(backup) + await db_session.commit() + await db_session.refresh(backup) + + # Mock validation returns no issues + mock_engine.validate_backup_prerequisites = AsyncMock(return_value=[]) + + response = await async_client.post(f"/api/v1/backups/{backup.id}/validate") + assert response.status_code == 200 + + data = response.json() + assert data["valid"] is True + assert data["issues"] == [] + + @patch('app.api.backups.backup_engine') + async def test_validate_backup_with_issues(self, mock_engine, async_client: AsyncClient, db_session): + """Test validating backup that has issues.""" + # Create target and backup + target = BackupTarget( + name="test-target", + target_type="volume", + volume_name="missing-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.PENDING, + backup_metadata={"target_name": "test-target"} + ) + db_session.add(backup) + await db_session.commit() + await db_session.refresh(backup) + + # Mock validation returns issues + mock_engine.validate_backup_prerequisites = AsyncMock( + return_value=["Volume 'missing-volume' not found", "Low disk space"] + ) + + response = await async_client.post(f"/api/v1/backups/{backup.id}/validate") + assert response.status_code == 200 + + data = response.json() + assert data["valid"] is False + assert len(data["issues"]) == 2 + assert "Volume 'missing-volume' not found" in data["issues"] + assert "Low disk space" in data["issues"] + + async def test_validate_backup_not_found(self, async_client: AsyncClient): + """Test validating non-existent backup.""" + response = await async_client.post("/api/v1/backups/99999/validate") + assert response.status_code == 404 + assert "Backup not found" in response.json()["detail"] diff --git a/backend/tests/test_backup_engine.py b/backend/tests/test_backup_engine.py new file mode 100644 index 0000000..850461c --- /dev/null +++ b/backend/tests/test_backup_engine.py @@ -0,0 +1,752 @@ +""" +Tests for backup_engine module. +""" + +import asyncio +import os +import shutil +import tempfile +from datetime import datetime +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch, mock_open +import pytest +from sqlalchemy import select + +from app.backup_engine import BackupEngine, BackupMetrics +from app.database import Backup, BackupTarget, BackupStatus, BackupType + + +@pytest.mark.asyncio +class TestBackupEngine: + """Test backup engine functionality.""" + + async def test_create_backup(self, db_session): + """Test backup creation.""" + # Create a target + target = BackupTarget( + name="test-volume", + target_type="volume", + volume_name="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + engine = BackupEngine() + backup = await engine.create_backup(target, BackupType.FULL) + + assert backup is not None + assert backup.target_id == target.id + assert backup.backup_type == BackupType.FULL + assert backup.status == BackupStatus.PENDING + assert backup.backup_metadata["target_name"] == "test-volume" + assert backup.backup_metadata["target_type"] == "volume" + + async def test_add_remove_progress_callback(self): + """Test progress callback management.""" + engine = BackupEngine() + callback = AsyncMock() + + # Add callback + engine.add_progress_callback(callback) + assert callback in engine.progress_callbacks + + # Test notification + await engine._notify_progress(1, 50.0, "Test progress") + callback.assert_called_once_with(1, 50.0, "Test progress") + + # Remove callback + engine.remove_progress_callback(callback) + assert callback not in engine.progress_callbacks + + async def test_notify_progress_with_exception(self): + """Test progress notification handles callback exceptions.""" + engine = BackupEngine() + failing_callback = AsyncMock(side_effect=Exception("Callback error")) + working_callback = AsyncMock() + + engine.add_progress_callback(failing_callback) + engine.add_progress_callback(working_callback) + + # Should not raise exception even if callback fails + await engine._notify_progress(1, 50.0, "Test progress") + + failing_callback.assert_called_once() + working_callback.assert_called_once() + + @patch('app.backup_engine.docker_client') + async def test_run_backup_volume(self, mock_docker, db_session, temp_backup_dir): + """Test running a volume backup.""" + # Setup mock Docker client + mock_volume = MagicMock() + mock_volume.name = "test-volume" + mock_docker.volumes.get.return_value = mock_volume + mock_docker.containers.create.return_value = MagicMock(id="container123") + mock_docker.containers.get.return_value = MagicMock() + + # Create target and backup + target = BackupTarget( + name="test-volume", + target_type="volume", + volume_name="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.PENDING, + backup_metadata={"target_name": "test-volume", "target_type": "volume"} + ) + db_session.add(backup) + await db_session.commit() + await db_session.refresh(backup) + + engine = BackupEngine() + + with patch('app.backup_engine.settings') as mock_settings: + mock_settings.BACKUP_ROOT = str(temp_backup_dir) + with patch('tarfile.open') as mock_tarfile: + mock_tar = MagicMock() + mock_tarfile.return_value.__enter__.return_value = mock_tar + + result = await engine.run_backup(backup.id) + + assert result is True + mock_docker.volumes.get.assert_called_with("test-volume") + mock_docker.containers.create.assert_called_once() + + async def test_run_backup_nonexistent(self, db_session): + """Test running backup for non-existent backup ID.""" + engine = BackupEngine() + result = await engine.run_backup(99999) + assert result is False + + async def test_run_backup_no_target(self, db_session): + """Test running backup with missing target.""" + # Create backup without target + backup = Backup( + target_id=99999, # Non-existent target + backup_type=BackupType.FULL, + status=BackupStatus.PENDING, + backup_metadata={} + ) + db_session.add(backup) + await db_session.commit() + await db_session.refresh(backup) + + engine = BackupEngine() + result = await engine.run_backup(backup.id) + assert result is False + + @patch('app.backup_engine.docker_client') + async def test_run_backup_docker_error(self, mock_docker, db_session): + """Test backup handles Docker errors gracefully.""" + # Setup Docker error + mock_docker.volumes.get.side_effect = Exception("Docker error") + + # Create target and backup + target = BackupTarget( + name="test-volume", + target_type="volume", + volume_name="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.PENDING, + backup_metadata={"target_name": "test-volume", "target_type": "volume"} + ) + db_session.add(backup) + await db_session.commit() + await db_session.refresh(backup) + + engine = BackupEngine() + result = await engine.run_backup(backup.id, skip_validation=True) + + assert result is False + + # Verify backup status was updated to failed + await db_session.refresh(backup) + assert backup.status == BackupStatus.FAILED + assert "Docker error" in backup.error_message + + async def test_security_path_traversal_prevention(self, db_session): + """Test that path traversal attempts are blocked.""" + # Create target with malicious path + target = BackupTarget( + name="../../../etc/passwd", + target_type="path", + host_path="../../../etc/passwd", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + engine = BackupEngine() + backup = await engine.create_backup(target) + + # Should reject path traversal attempts + result = await engine.run_backup(backup.id) + assert result is False + + await db_session.refresh(backup) + assert backup.status == BackupStatus.FAILED + + +@pytest.mark.asyncio +class TestBackupMetrics: + """Test BackupMetrics functionality.""" + + def test_metrics_initial_state(self): + """Test initial metrics state.""" + metrics = BackupMetrics() + + assert metrics.total_backups == 0 + assert metrics.successful_backups == 0 + assert metrics.failed_backups == 0 + assert metrics.total_bytes_backed_up == 0 + assert metrics.success_rate == 0.0 + assert metrics.last_backup_time is None + + def test_record_successful_backup(self): + """Test recording a successful backup.""" + metrics = BackupMetrics() + + metrics.record_backup( + target_id=1, + duration=120.5, + size=1024000, + success=True, + ) + + assert metrics.total_backups == 1 + assert metrics.successful_backups == 1 + assert metrics.failed_backups == 0 + assert metrics.total_bytes_backed_up == 1024000 + assert metrics.success_rate == 100.0 + assert metrics.last_backup_time is not None + + def test_record_failed_backup(self): + """Test recording a failed backup.""" + metrics = BackupMetrics() + + metrics.record_backup( + target_id=1, + duration=30.0, + size=0, + success=False, + ) + + assert metrics.total_backups == 1 + assert metrics.successful_backups == 0 + assert metrics.failed_backups == 1 + assert metrics.total_bytes_backed_up == 0 + assert metrics.success_rate == 0.0 + + def test_success_rate_calculation(self): + """Test success rate calculation with mixed results.""" + metrics = BackupMetrics() + + # Record 3 successful, 1 failed + for _ in range(3): + metrics.record_backup(target_id=1, duration=60.0, size=1000, success=True) + metrics.record_backup(target_id=1, duration=30.0, size=0, success=False) + + assert metrics.total_backups == 4 + assert metrics.successful_backups == 3 + assert metrics.failed_backups == 1 + assert metrics.success_rate == 75.0 + + def test_average_duration_calculation(self): + """Test average duration calculation per target.""" + metrics = BackupMetrics() + + # Record backups with different durations + metrics.record_backup(target_id=1, duration=60.0, size=1000, success=True) + metrics.record_backup(target_id=1, duration=120.0, size=1000, success=True) + metrics.record_backup(target_id=1, duration=90.0, size=1000, success=True) + + avg = metrics.get_average_duration(1) + assert avg == 90.0 # (60 + 120 + 90) / 3 + + def test_average_size_calculation(self): + """Test average size calculation per target.""" + metrics = BackupMetrics() + + metrics.record_backup(target_id=2, duration=60.0, size=1000, success=True) + metrics.record_backup(target_id=2, duration=60.0, size=2000, success=True) + metrics.record_backup(target_id=2, duration=60.0, size=3000, success=True) + + avg = metrics.get_average_size(2) + assert avg == 2000 # (1000 + 2000 + 3000) / 3 + + def test_metrics_per_target_isolation(self): + """Test that metrics are isolated per target.""" + metrics = BackupMetrics() + + metrics.record_backup(target_id=1, duration=60.0, size=1000, success=True) + metrics.record_backup(target_id=2, duration=120.0, size=2000, success=True) + + assert metrics.get_average_duration(1) == 60.0 + assert metrics.get_average_duration(2) == 120.0 + assert metrics.get_average_size(1) == 1000 + assert metrics.get_average_size(2) == 2000 + + def test_nonexistent_target_returns_none(self): + """Test that non-existent target returns None for averages.""" + metrics = BackupMetrics() + + assert metrics.get_average_duration(999) is None + assert metrics.get_average_size(999) is None + + def test_to_dict_serialization(self): + """Test metrics serialization to dict.""" + metrics = BackupMetrics() + metrics.record_backup(target_id=1, duration=60.0, size=1000, success=True) + + result = metrics.to_dict() + + assert "total_backups" in result + assert "successful_backups" in result + assert "failed_backups" in result + assert "success_rate" in result + assert "total_bytes_backed_up" in result + assert "last_backup_time" in result + + assert result["total_backups"] == 1 + assert result["success_rate"] == 100.0 + + def test_metrics_memory_limit(self): + """Test that metrics limit stored history per target.""" + metrics = BackupMetrics() + + # Record more than 100 backups for same target + for i in range(150): + metrics.record_backup(target_id=1, duration=float(i), size=i * 100, success=True) + + # Should only keep last 100 + assert len(metrics.target_durations[1]) == 100 + assert len(metrics.target_sizes[1]) == 100 + + +@pytest.mark.asyncio +class TestBackupValidation: + """Test backup validation functionality.""" + + @patch('app.backup_engine.docker_client') + async def test_validate_missing_container(self, mock_docker, db_session): + """Test validation fails when container doesn't exist.""" + # Mock empty container list + mock_docker.list_containers = AsyncMock(return_value=[]) + + target = BackupTarget( + name="test-container", + target_type="container", + container_name="missing-container", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + engine = BackupEngine() + issues = await engine.validate_backup_prerequisites(target) + + assert len(issues) > 0 + assert any("not found" in issue.lower() for issue in issues) + + @patch('app.backup_engine.docker_client') + async def test_validate_missing_volume(self, mock_docker, db_session): + """Test validation fails when volume doesn't exist.""" + # Mock empty volume list + mock_docker.list_volumes = AsyncMock(return_value=[]) + + target = BackupTarget( + name="test-volume", + target_type="volume", + volume_name="missing-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + engine = BackupEngine() + issues = await engine.validate_backup_prerequisites(target) + + assert len(issues) > 0 + assert any("not found" in issue.lower() for issue in issues) + + async def test_validate_missing_path(self, db_session): + """Test validation fails when path doesn't exist.""" + target = BackupTarget( + name="test-path", + target_type="path", + host_path="/nonexistent/path/to/backup", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + engine = BackupEngine() + issues = await engine.validate_backup_prerequisites(target) + + assert len(issues) > 0 + assert any("not found" in issue.lower() for issue in issues) + + @patch('app.backup_engine.docker_client') + async def test_validate_missing_dependency(self, mock_docker, db_session): + """Test validation fails when dependency container doesn't exist.""" + # Mock empty container list + mock_docker.list_containers = AsyncMock(return_value=[]) + mock_docker.list_volumes = AsyncMock(return_value=[ + MagicMock(name="test-volume") + ]) + + target = BackupTarget( + name="test-volume", + target_type="volume", + volume_name="test-volume", + dependencies=["missing-dependency"], + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + engine = BackupEngine() + issues = await engine.validate_backup_prerequisites(target) + + assert len(issues) > 0 + assert any("dependency" in issue.lower() for issue in issues) + + @patch('shutil.disk_usage') + @patch('app.backup_engine.docker_client') + async def test_validate_low_disk_space(self, mock_docker, mock_disk_usage, db_session, temp_backup_dir): + """Test validation warns on low disk space.""" + # Mock low disk space (500MB free) + mock_disk_usage.return_value = MagicMock( + free=500 * 1024 * 1024, # 500 MB + total=100 * 1024 * 1024 * 1024, # 100 GB + used=99.5 * 1024 * 1024 * 1024, # 99.5 GB + ) + + # Mock volume exists + mock_volume = MagicMock() + mock_volume.name = "test-volume" + mock_docker.list_volumes = AsyncMock(return_value=[mock_volume]) + + target = BackupTarget( + name="test-volume", + target_type="volume", + volume_name="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + engine = BackupEngine() + + with patch('app.backup_engine.settings') as mock_settings: + mock_settings.BACKUP_BASE_PATH = str(temp_backup_dir) + issues = await engine.validate_backup_prerequisites(target) + + assert any("disk space" in issue.lower() for issue in issues) + + @patch('app.backup_engine.docker_client') + async def test_validate_existing_container(self, mock_docker, db_session): + """Test validation passes when container exists.""" + # Mock container exists + mock_container = MagicMock() + mock_container.name = "my-container" + mock_docker.list_containers = AsyncMock(return_value=[mock_container]) + + target = BackupTarget( + name="test-container", + target_type="container", + container_name="my-container", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + engine = BackupEngine() + + with patch('shutil.disk_usage') as mock_disk: + # Mock plenty of disk space + mock_disk.return_value = MagicMock(free=100 * 1024 * 1024 * 1024) + issues = await engine.validate_backup_prerequisites(target) + + # Should have no issues (or only non-critical ones) + assert not any("container" in issue.lower() and "not found" in issue.lower() for issue in issues) + + +@pytest.mark.asyncio +class TestBackupConcurrency: + """Test backup concurrency and semaphore functionality.""" + + async def test_backup_semaphore_initialization(self): + """Test that backup engine initializes with semaphore.""" + engine = BackupEngine() + + assert hasattr(engine, '_backup_semaphore') + assert isinstance(engine._backup_semaphore, asyncio.Semaphore) + + async def test_concurrent_backup_limit(self, db_session): + """Test that concurrent backups are limited by semaphore.""" + from app.config import settings + + engine = BackupEngine() + + # Create multiple targets and backups + backups = [] + for i in range(5): + target = BackupTarget( + name=f"test-volume-{i}", + target_type="volume", + volume_name=f"test-volume-{i}", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.PENDING, + backup_metadata={"target_name": f"test-volume-{i}", "target_type": "volume"} + ) + db_session.add(backup) + await db_session.commit() + await db_session.refresh(backup) + backups.append(backup) + + # Track concurrent execution count + concurrent_count = 0 + max_concurrent = 0 + lock = asyncio.Lock() + + original_run = engine.run_backup + + async def tracked_run(backup_id, skip_validation=False): + nonlocal concurrent_count, max_concurrent + async with lock: + concurrent_count += 1 + max_concurrent = max(max_concurrent, concurrent_count) + + # Simulate some work + await asyncio.sleep(0.1) + + async with lock: + concurrent_count -= 1 + + return False # Return False to avoid actual backup + + with patch.object(engine, 'run_backup', tracked_run): + # Run all backups concurrently + tasks = [engine.run_backup(b.id) for b in backups] + await asyncio.gather(*tasks) + + # Max concurrent should not exceed the limit + assert max_concurrent <= settings.MAX_CONCURRENT_BACKUPS + + async def test_metrics_tracked_after_backup(self, db_session): + """Test that metrics are updated after backup completion.""" + engine = BackupEngine() + + initial_total = engine.metrics.total_backups + + # Create target and backup + target = BackupTarget( + name="test-volume", + target_type="volume", + volume_name="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.PENDING, + backup_metadata={"target_name": "test-volume", "target_type": "volume"} + ) + db_session.add(backup) + await db_session.commit() + await db_session.refresh(backup) + + # Run backup (will fail due to missing volume, but should track metrics) + with patch('app.backup_engine.docker_client') as mock_docker: + mock_docker.list_volumes = AsyncMock(return_value=[]) + await engine.run_backup(backup.id) + + # Metrics should be updated + assert engine.metrics.total_backups > initial_total + + +@pytest.mark.asyncio +class TestBackupContainerDependencies: + """Test backup with container dependency management.""" + + @patch('app.backup_engine.docker_client') + async def test_backup_stops_containers_in_order(self, mock_docker, db_session): + """Test that containers are stopped in dependency order.""" + stopped_containers = [] + + async def mock_stop(name): + stopped_containers.append(name) + return True + + async def mock_start(name): + return True + + async def mock_get_state(name): + return "running" + + mock_docker.stop_container = mock_stop + mock_docker.start_container = mock_start + mock_docker.get_container_state = mock_get_state + mock_docker.get_dependency_order = AsyncMock( + return_value=["app", "db", "redis"] + ) + mock_docker.list_volumes = AsyncMock(return_value=[ + MagicMock(name="test-volume", mountpoint="/var/lib/docker/volumes/test-volume/_data") + ]) + + target = BackupTarget( + name="test-app", + target_type="volume", + volume_name="test-volume", + container_name="app", + stop_container=True, + dependencies=["db", "redis"], + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.PENDING, + backup_metadata={"target_name": "test-app", "target_type": "volume"} + ) + db_session.add(backup) + await db_session.commit() + await db_session.refresh(backup) + + engine = BackupEngine() + + with patch('app.backup_engine.settings') as mock_settings: + mock_settings.BACKUP_BASE_PATH = "/tmp/test_backups" + mock_settings.COMPRESSION_LEVEL = 6 + with patch.object(engine, '_create_backup_archive', AsyncMock(return_value="/tmp/backup.tar.gz")): + with patch.object(engine, '_calculate_checksum', AsyncMock(return_value="abc123")): + await engine.run_backup(backup.id, skip_validation=True) + + # Verify containers were stopped + assert "app" in stopped_containers or len(stopped_containers) > 0 + + +@pytest.mark.asyncio +class TestTarSecurityValidation: + """Test tar archive security validation.""" + + def test_extract_tar_blocks_path_traversal(self, temp_backup_dir): + """Test that path traversal in tar archives is blocked.""" + import tarfile + import io + + # Create a malicious tar file with path traversal + tar_path = temp_backup_dir / "malicious.tar" + with tarfile.open(tar_path, "w") as tar: + # Add a file with path traversal attempt + info = tarfile.TarInfo(name="../../../etc/passwd") + info.size = 4 + tar.addfile(info, io.BytesIO(b"test")) + + engine = BackupEngine() + extract_dir = temp_backup_dir / "extract" + extract_dir.mkdir() + + with pytest.raises(ValueError, match="[Pp]ath traversal"): + engine._extract_tar(str(tar_path), str(extract_dir), "r") + + def test_extract_tar_blocks_absolute_paths(self, temp_backup_dir): + """Test that absolute paths in tar archives are blocked.""" + import tarfile + import io + + tar_path = temp_backup_dir / "absolute.tar" + with tarfile.open(tar_path, "w") as tar: + # Add a file with absolute path + info = tarfile.TarInfo(name="/etc/passwd") + info.size = 4 + tar.addfile(info, io.BytesIO(b"test")) + + engine = BackupEngine() + extract_dir = temp_backup_dir / "extract" + extract_dir.mkdir() + + with pytest.raises(ValueError, match="[Aa]bsolute path"): + engine._extract_tar(str(tar_path), str(extract_dir), "r") + + def test_extract_tar_blocks_symlink_escape(self, temp_backup_dir): + """Test that symlink escape attempts in tar archives are blocked.""" + import tarfile + + tar_path = temp_backup_dir / "symlink.tar" + with tarfile.open(tar_path, "w") as tar: + # Add a symlink pointing outside the extract directory + info = tarfile.TarInfo(name="link") + info.type = tarfile.SYMTYPE + info.linkname = "../../../etc/passwd" + tar.addfile(info) + + engine = BackupEngine() + extract_dir = temp_backup_dir / "extract" + extract_dir.mkdir() + + with pytest.raises(ValueError, match="[Ss]ymlink escape"): + engine._extract_tar(str(tar_path), str(extract_dir), "r") + + def test_extract_tar_allows_safe_files(self, temp_backup_dir): + """Test that safe tar files are extracted correctly.""" + import tarfile + import io + + tar_path = temp_backup_dir / "safe.tar" + with tarfile.open(tar_path, "w") as tar: + # Add safe files + info = tarfile.TarInfo(name="data/file.txt") + content = b"Hello, World!" + info.size = len(content) + tar.addfile(info, io.BytesIO(content)) + + engine = BackupEngine() + extract_dir = temp_backup_dir / "extract" + extract_dir.mkdir() + + # Should not raise + engine._extract_tar(str(tar_path), str(extract_dir), "r") + + # Verify file was extracted + assert (extract_dir / "data" / "file.txt").exists() diff --git a/backend/tests/test_database.py b/backend/tests/test_database.py new file mode 100644 index 0000000..218a4fe --- /dev/null +++ b/backend/tests/test_database.py @@ -0,0 +1,346 @@ +""" +Tests for database module. +""" + +import pytest +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from datetime import datetime + +from app.database import ( + Backup, BackupTarget, BackupSchedule, RemoteStorageConfig, + BackupStatus, BackupType, TargetType, StorageType, ScheduleType +) + + +@pytest.mark.asyncio +class TestDatabaseModels: + """Test database models and relationships.""" + + async def test_backup_target_creation(self, db_session): + """Test backup target creation.""" + target = BackupTarget( + name="test-volume", + target_type=TargetType.VOLUME, + source_path="test-volume", + enabled=True, + description="Test volume for backup" + ) + + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + assert target.id is not None + assert target.name == "test-volume" + assert target.target_type == TargetType.VOLUME + assert target.enabled is True + assert target.created_at is not None + + async def test_backup_target_unique_name_constraint(self, db_session): + """Test unique name constraint on backup targets.""" + # Create first target + target1 = BackupTarget( + name="duplicate-name", + target_type=TargetType.VOLUME, + source_path="volume1", + enabled=True + ) + db_session.add(target1) + await db_session.commit() + + # Try to create second target with same name + target2 = BackupTarget( + name="duplicate-name", + target_type=TargetType.HOST_PATH, + source_path="/path/to/data", + enabled=True + ) + db_session.add(target2) + + with pytest.raises(IntegrityError): + await db_session.commit() + + async def test_backup_creation(self, db_session): + """Test backup creation with target relationship.""" + # Create target first + target = BackupTarget( + name="test-target", + target_type=TargetType.VOLUME, + source_path="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + # Create backup + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.PENDING, + backup_metadata={"test": "data"}, + file_path="/backups/test.tar.gz", + file_size=1024, + checksum="abc123" + ) + + db_session.add(backup) + await db_session.commit() + await db_session.refresh(backup) + + assert backup.id is not None + assert backup.target_id == target.id + assert backup.backup_type == BackupType.FULL + assert backup.status == BackupStatus.PENDING + assert backup.metadata == {"test": "data"} + assert backup.created_at is not None + + async def test_backup_target_relationship(self, db_session): + """Test backup-target relationship loading.""" + # Create target + target = BackupTarget( + name="relationship-test", + target_type=TargetType.VOLUME, + source_path="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + # Create multiple backups for target + backup1 = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.COMPLETED + ) + backup2 = Backup( + target_id=target.id, + backup_type=BackupType.INCREMENTAL, + status=BackupStatus.PENDING + ) + + db_session.add_all([backup1, backup2]) + await db_session.commit() + + # Query target with backups + result = await db_session.execute( + select(BackupTarget).where(BackupTarget.id == target.id) + ) + loaded_target = result.scalar_one() + + # Note: In real app, you'd use relationship loading + # This tests the foreign key relationship exists + backup_result = await db_session.execute( + select(Backup).where(Backup.target_id == target.id) + ) + target_backups = backup_result.scalars().all() + + assert len(target_backups) == 2 + assert all(b.target_id == target.id for b in target_backups) + + async def test_backup_schedule_creation(self, db_session): + """Test backup schedule creation.""" + # Create target first + target = BackupTarget( + name="scheduled-target", + target_type=TargetType.VOLUME, + source_path="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + # Create schedule + schedule = BackupSchedule( + name="Daily Backup", + target_id=target.id, + schedule_type=ScheduleType.CRON, + cron_expression="0 2 * * *", # Daily at 2 AM + backup_type=BackupType.FULL, + enabled=True, + retention_days=30 + ) + + db_session.add(schedule) + await db_session.commit() + await db_session.refresh(schedule) + + assert schedule.id is not None + assert schedule.name == "Daily Backup" + assert schedule.target_id == target.id + assert schedule.schedule_type == ScheduleType.CRON + assert schedule.cron_expression == "0 2 * * *" + assert schedule.enabled is True + assert schedule.retention_days == 30 + + async def test_remote_storage_config_creation(self, db_session): + """Test remote storage configuration.""" + storage_config = RemoteStorageConfig( + name="S3 Storage", + storage_type=StorageType.S3, + config={ + "bucket": "my-backup-bucket", + "region": "us-east-1", + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + }, + enabled=True + ) + + db_session.add(storage_config) + await db_session.commit() + await db_session.refresh(storage_config) + + assert storage_config.id is not None + assert storage_config.name == "S3 Storage" + assert storage_config.storage_type == StorageType.S3 + assert storage_config.config["bucket"] == "my-backup-bucket" + assert storage_config.enabled is True + + async def test_backup_status_transitions(self, db_session): + """Test backup status transitions.""" + # Create target + target = BackupTarget( + name="status-test", + target_type=TargetType.VOLUME, + source_path="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + # Create backup + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.PENDING + ) + db_session.add(backup) + await db_session.commit() + await db_session.refresh(backup) + + # Update status to running + backup.status = BackupStatus.RUNNING + backup.started_at = datetime.utcnow() + await db_session.commit() + + # Update status to completed + backup.status = BackupStatus.COMPLETED + backup.completed_at = datetime.utcnow() + backup.file_size = 2048 + backup.checksum = "def456" + await db_session.commit() + + await db_session.refresh(backup) + + assert backup.status == BackupStatus.COMPLETED + assert backup.started_at is not None + assert backup.completed_at is not None + assert backup.file_size == 2048 + assert backup.checksum == "def456" + + async def test_database_connection_error_handling(self, db_session): + """Test database connection error handling.""" + # This test checks that database operations handle errors gracefully + # In real scenarios, this might involve network issues, disk full, etc. + + # Create a backup with invalid target_id (foreign key constraint) + backup = Backup( + target_id=99999, # Non-existent target + backup_type=BackupType.FULL, + status=BackupStatus.PENDING + ) + + db_session.add(backup) + + with pytest.raises(IntegrityError): + await db_session.commit() + + async def test_metadata_json_field(self, db_session): + """Test JSON metadata field functionality.""" + target = BackupTarget( + name="json-test", + target_type=TargetType.VOLUME, + source_path="test-volume", + enabled=True + ) + db_session.add(target) + await db_session.commit() + await db_session.refresh(target) + + # Test complex metadata structure + complex_metadata = { + "containers_stopped": ["container1", "container2"], + "backup_settings": { + "compression": "gzip", + "encryption": False, + "exclude_patterns": ["*.log", "tmp/*"] + }, + "performance": { + "start_time": "2024-01-01T10:00:00Z", + "duration_seconds": 120, + "bytes_processed": 1048576 + } + } + + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.COMPLETED, + backup_metadata=complex_metadata + ) + + db_session.add(backup) + await db_session.commit() + await db_session.refresh(backup) + + assert backup.backup_metadata["containers_stopped"] == ["container1", "container2"] + assert backup.backup_metadata["backup_settings"]["compression"] == "gzip" + assert backup.backup_metadata["performance"]["bytes_processed"] == 1048576 + + async def test_query_performance_indexes(self, db_session): + """Test that database indexes work correctly for common queries.""" + # Create multiple targets and backups + targets = [] + for i in range(10): + target = BackupTarget( + name=f"target-{i}", + target_type=TargetType.VOLUME, + source_path=f"volume-{i}", + enabled=True + ) + targets.append(target) + db_session.add(target) + + await db_session.commit() + + # Create many backups + for target in targets: + await db_session.refresh(target) + for j in range(5): + backup = Backup( + target_id=target.id, + backup_type=BackupType.FULL, + status=BackupStatus.COMPLETED if j < 3 else BackupStatus.FAILED + ) + db_session.add(backup) + + await db_session.commit() + + # Query by status (should use index) + completed_result = await db_session.execute( + select(Backup).where(Backup.status == BackupStatus.COMPLETED) + ) + completed_backups = completed_result.scalars().all() + assert len(completed_backups) == 30 # 10 targets * 3 completed each + + # Query by target_id (should use foreign key index) + target_result = await db_session.execute( + select(Backup).where(Backup.target_id == targets[0].id) + ) + target_backups = target_result.scalars().all() + assert len(target_backups) == 5 diff --git a/backend/tests/test_docker_client.py b/backend/tests/test_docker_client.py new file mode 100644 index 0000000..42ea43d --- /dev/null +++ b/backend/tests/test_docker_client.py @@ -0,0 +1,264 @@ +""" +Tests for docker_client module. +""" + +from unittest.mock import MagicMock, patch +import pytest +from docker.errors import DockerException, NotFound, APIError + +from app.docker_client import DockerClientWrapper, ContainerInfo + + +@pytest.mark.asyncio +class TestDockerClientWrapper: + """Test Docker client wrapper functionality.""" + + @patch('app.docker_client.docker.from_env') + def test_init_success(self, mock_docker): + """Test successful Docker client initialization.""" + mock_client = MagicMock() + mock_docker.return_value = mock_client + mock_client.ping.return_value = True + + wrapper = DockerClientWrapper() + assert wrapper.client is not None + assert wrapper.available is True + mock_client.ping.assert_called_once() + + @patch('app.docker_client.docker.from_env') + def test_init_failure(self, mock_docker): + """Test Docker client initialization failure.""" + mock_docker.side_effect = DockerException("Docker not available") + + wrapper = DockerClientWrapper() + assert wrapper.client is None + assert wrapper.available is False + + @patch('app.docker_client.docker.from_env') + def test_get_containers_success(self, mock_docker): + """Test successful container listing.""" + mock_client = MagicMock() + mock_docker.return_value = mock_client + mock_client.ping.return_value = True + + # Mock container data + mock_container = MagicMock() + mock_container.id = "container123" + mock_container.name = "test-container" + mock_container.image.tags = ["nginx:latest"] + mock_container.status = "running" + mock_container.attrs = { + "State": {"Status": "running"}, + "Created": "2024-01-01T00:00:00Z", + "Config": {"Labels": {"com.docker.compose.project": "myproject"}}, + "Mounts": [], + "NetworkSettings": {"Networks": {"bridge": {}}} + } + mock_client.containers.list.return_value = [mock_container] + + wrapper = DockerClientWrapper() + containers = wrapper.get_containers() + + assert len(containers) == 1 + container = containers[0] + assert isinstance(container, ContainerInfo) + assert container.id == "container123" + assert container.name == "test-container" + assert container.image == "nginx:latest" + assert container.status == "running" + assert container.compose_project == "myproject" + + @patch('app.docker_client.docker.from_env') + def test_get_containers_docker_unavailable(self, mock_docker): + """Test container listing when Docker is unavailable.""" + mock_docker.side_effect = DockerException("Docker not available") + + wrapper = DockerClientWrapper() + containers = wrapper.get_containers() + + assert containers == [] + + @patch('app.docker_client.docker.from_env') + def test_get_volumes_success(self, mock_docker): + """Test successful volume listing.""" + mock_client = MagicMock() + mock_docker.return_value = mock_client + mock_client.ping.return_value = True + + # Mock volume data + mock_volume = MagicMock() + mock_volume.name = "test-volume" + mock_volume.attrs = { + "Driver": "local", + "Mountpoint": "/var/lib/docker/volumes/test-volume/_data", + "Labels": {}, + "CreatedAt": "2024-01-01T00:00:00Z" + } + mock_client.volumes.list.return_value = [mock_volume] + + # Mock containers using volume + mock_container = MagicMock() + mock_container.name = "container1" + mock_container.attrs = { + "Mounts": [{ + "Type": "volume", + "Name": "test-volume", + "Source": "/var/lib/docker/volumes/test-volume/_data", + "Destination": "/data" + }] + } + mock_client.containers.list.return_value = [mock_container] + + wrapper = DockerClientWrapper() + volumes = wrapper.get_volumes() + + assert len(volumes) == 1 + volume = volumes[0] + assert volume["name"] == "test-volume" + assert volume["driver"] == "local" + assert "container1" in volume["used_by"] + + @patch('app.docker_client.docker.from_env') + def test_stop_containers_success(self, mock_docker): + """Test successful container stopping.""" + mock_client = MagicMock() + mock_docker.return_value = mock_client + mock_client.ping.return_value = True + + mock_container = MagicMock() + mock_container.status = "running" + mock_client.containers.get.return_value = mock_container + + wrapper = DockerClientWrapper() + success = wrapper.stop_containers(["container1"]) + + assert success is True + mock_client.containers.get.assert_called_with("container1") + mock_container.stop.assert_called_once() + + @patch('app.docker_client.docker.from_env') + def test_stop_containers_not_found(self, mock_docker): + """Test stopping non-existent container.""" + mock_client = MagicMock() + mock_docker.return_value = mock_client + mock_client.ping.return_value = True + + mock_client.containers.get.side_effect = NotFound("Container not found") + + wrapper = DockerClientWrapper() + success = wrapper.stop_containers(["nonexistent"]) + + assert success is False + + @patch('app.docker_client.docker.from_env') + def test_start_containers_success(self, mock_docker): + """Test successful container starting.""" + mock_client = MagicMock() + mock_docker.return_value = mock_client + mock_client.ping.return_value = True + + mock_container = MagicMock() + mock_container.status = "exited" + mock_client.containers.get.return_value = mock_container + + wrapper = DockerClientWrapper() + success = wrapper.start_containers(["container1"]) + + assert success is True + mock_container.start.assert_called_once() + + @patch('app.docker_client.docker.from_env') + def test_create_backup_container(self, mock_docker): + """Test backup container creation.""" + mock_client = MagicMock() + mock_docker.return_value = mock_client + mock_client.ping.return_value = True + + mock_container = MagicMock() + mock_container.id = "backup123" + mock_client.containers.create.return_value = mock_container + + wrapper = DockerClientWrapper() + container_id = wrapper.create_backup_container( + "test-volume", + "/backup/test.tar.gz" + ) + + assert container_id == "backup123" + mock_client.containers.create.assert_called_once() + + # Verify container creation parameters + call_args = mock_client.containers.create.call_args + assert "ubuntu:latest" in call_args[1]["image"] + assert "/data" in str(call_args[1]["volumes"]) + + @patch('app.docker_client.docker.from_env') + def test_security_volume_name_validation(self, mock_docker): + """Test that malicious volume names are rejected.""" + mock_client = MagicMock() + mock_docker.return_value = mock_client + mock_client.ping.return_value = True + + wrapper = DockerClientWrapper() + + # Test path traversal in volume name + with pytest.raises(ValueError, match="Invalid volume name"): + wrapper.create_backup_container( + "../../../etc/passwd", + "/backup/test.tar.gz" + ) + + # Test command injection in volume name + with pytest.raises(ValueError, match="Invalid volume name"): + wrapper.create_backup_container( + "volume; rm -rf /", + "/backup/test.tar.gz" + ) + + @patch('app.docker_client.docker.from_env') + def test_wait_for_container_success(self, mock_docker): + """Test successful container wait.""" + mock_client = MagicMock() + mock_docker.return_value = mock_client + mock_client.ping.return_value = True + + mock_container = MagicMock() + mock_container.wait.return_value = {"StatusCode": 0} + mock_container.logs.return_value = b"Backup completed" + mock_client.containers.get.return_value = mock_container + + wrapper = DockerClientWrapper() + exit_code, logs = wrapper.wait_for_container("container123") + + assert exit_code == 0 + assert logs == "Backup completed" + mock_container.wait.assert_called_once() + mock_container.logs.assert_called_once() + + @patch('app.docker_client.docker.from_env') + def test_cleanup_container(self, mock_docker): + """Test container cleanup.""" + mock_client = MagicMock() + mock_docker.return_value = mock_client + mock_client.ping.return_value = True + + mock_container = MagicMock() + mock_client.containers.get.return_value = mock_container + + wrapper = DockerClientWrapper() + wrapper.cleanup_container("container123") + + mock_container.remove.assert_called_once_with(force=True) + + @patch('app.docker_client.docker.from_env') + def test_cleanup_container_not_found(self, mock_docker): + """Test cleanup of non-existent container.""" + mock_client = MagicMock() + mock_docker.return_value = mock_client + mock_client.ping.return_value = True + + mock_client.containers.get.side_effect = NotFound("Container not found") + + wrapper = DockerClientWrapper() + # Should not raise exception + wrapper.cleanup_container("nonexistent") diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py new file mode 100644 index 0000000..9a523c1 --- /dev/null +++ b/backend/tests/test_scheduler.py @@ -0,0 +1,422 @@ +""" +Tests for scheduler module. +""" + +import asyncio +from datetime import datetime, timedelta +from unittest.mock import AsyncMock, MagicMock, patch +import pytest +from apscheduler.job import Job + +from app.scheduler import BackupScheduler +from app.database import BackupTarget, BackupSchedule, TargetType, ScheduleType, BackupType + + +@pytest.mark.asyncio +class TestBackupScheduler: + """Test backup scheduler functionality.""" + + async def test_scheduler_initialization(self): + """Test scheduler initialization.""" + scheduler = BackupScheduler() + assert scheduler is not None + assert scheduler.scheduler is not None + assert not scheduler.scheduler.running + + async def test_scheduler_start_stop(self): + """Test scheduler start and stop.""" + scheduler = BackupScheduler() + + # Start scheduler + await scheduler.start() + assert scheduler.scheduler.running + + # Stop scheduler + await scheduler.stop() + assert not scheduler.scheduler.running + + @patch('app.scheduler.async_session') + async def test_load_schedules_from_database(self, mock_session): + """Test loading schedules from database.""" + # Mock database session and query results + mock_session_instance = AsyncMock() + mock_session.return_value.__aenter__.return_value = mock_session_instance + + # Mock target + target = BackupTarget( + id=1, + name="test-volume", + target_type=TargetType.VOLUME, + source_path="test-volume", + enabled=True + ) + + # Mock schedule + schedule = BackupSchedule( + id=1, + name="Daily Backup", + target_id=1, + schedule_type=ScheduleType.CRON, + cron_expression="0 2 * * *", + backup_type=BackupType.FULL, + enabled=True, + retention_days=30 + ) + schedule.target = target # Set relationship + + # Mock query result + mock_result = MagicMock() + mock_result.scalars.return_value.all.return_value = [schedule] + mock_session_instance.execute.return_value = mock_result + + scheduler = BackupScheduler() + await scheduler.start() + + # Verify schedule was loaded + jobs = scheduler.scheduler.get_jobs() + assert len(jobs) > 0 + + await scheduler.stop() + + @patch('app.scheduler.backup_engine') + async def test_execute_scheduled_backup(self, mock_backup_engine): + """Test execution of scheduled backup.""" + mock_backup_engine.create_backup.return_value = AsyncMock() + mock_backup_engine.run_backup.return_value = AsyncMock(return_value=True) + + scheduler = BackupScheduler() + + # Create mock target + target = BackupTarget( + id=1, + name="test-volume", + target_type=TargetType.VOLUME, + source_path="test-volume", + enabled=True + ) + + # Execute backup + await scheduler._execute_backup(target, BackupType.FULL) + + # Verify backup engine was called + mock_backup_engine.create_backup.assert_called_once_with(target, BackupType.FULL) + mock_backup_engine.run_backup.assert_called_once() + + @patch('app.scheduler.backup_engine') + async def test_execute_backup_handles_errors(self, mock_backup_engine): + """Test that backup execution handles errors gracefully.""" + mock_backup_engine.create_backup.side_effect = Exception("Backup failed") + + scheduler = BackupScheduler() + + target = BackupTarget( + id=1, + name="test-volume", + target_type=TargetType.VOLUME, + source_path="test-volume", + enabled=True + ) + + # Should not raise exception + await scheduler._execute_backup(target, BackupType.FULL) + + # Verify error was logged (would check logs in real implementation) + mock_backup_engine.create_backup.assert_called_once() + + async def test_add_job_to_scheduler(self): + """Test adding job to scheduler.""" + scheduler = BackupScheduler() + await scheduler.start() + + target = BackupTarget( + id=1, + name="test-volume", + target_type=TargetType.VOLUME, + source_path="test-volume", + enabled=True + ) + + # Add job + job_id = await scheduler.add_job( + target=target, + cron_expression="0 2 * * *", + backup_type=BackupType.FULL + ) + + assert job_id is not None + + # Verify job was added + jobs = scheduler.scheduler.get_jobs() + job_ids = [job.id for job in jobs] + assert job_id in job_ids + + await scheduler.stop() + + async def test_remove_job_from_scheduler(self): + """Test removing job from scheduler.""" + scheduler = BackupScheduler() + await scheduler.start() + + target = BackupTarget( + id=1, + name="test-volume", + target_type=TargetType.VOLUME, + source_path="test-volume", + enabled=True + ) + + # Add job + job_id = await scheduler.add_job( + target=target, + cron_expression="0 2 * * *", + backup_type=BackupType.FULL + ) + + # Verify job exists + job = scheduler.scheduler.get_job(job_id) + assert job is not None + + # Remove job + await scheduler.remove_job(job_id) + + # Verify job was removed + job = scheduler.scheduler.get_job(job_id) + assert job is None + + await scheduler.stop() + + async def test_update_job_in_scheduler(self): + """Test updating job in scheduler.""" + scheduler = BackupScheduler() + await scheduler.start() + + target = BackupTarget( + id=1, + name="test-volume", + target_type=TargetType.VOLUME, + source_path="test-volume", + enabled=True + ) + + # Add job + job_id = await scheduler.add_job( + target=target, + cron_expression="0 2 * * *", # Daily at 2 AM + backup_type=BackupType.FULL + ) + + # Update job with new cron expression + await scheduler.update_job( + job_id=job_id, + cron_expression="0 6 * * *", # Daily at 6 AM + backup_type=BackupType.INCREMENTAL + ) + + # Verify job was updated + job = scheduler.scheduler.get_job(job_id) + assert job is not None + # In real implementation, would verify cron expression was updated + + await scheduler.stop() + + async def test_list_scheduled_jobs(self): + """Test listing scheduled jobs.""" + scheduler = BackupScheduler() + await scheduler.start() + + target1 = BackupTarget( + id=1, + name="test-volume-1", + target_type=TargetType.VOLUME, + source_path="test-volume-1", + enabled=True + ) + + target2 = BackupTarget( + id=2, + name="test-volume-2", + target_type=TargetType.VOLUME, + source_path="test-volume-2", + enabled=True + ) + + # Add multiple jobs + job_id1 = await scheduler.add_job( + target=target1, + cron_expression="0 2 * * *", + backup_type=BackupType.FULL + ) + + job_id2 = await scheduler.add_job( + target=target2, + cron_expression="0 6 * * *", + backup_type=BackupType.INCREMENTAL + ) + + # List jobs + jobs = await scheduler.list_jobs() + + assert len(jobs) >= 2 + job_ids = [job['id'] for job in jobs] + assert job_id1 in job_ids + assert job_id2 in job_ids + + await scheduler.stop() + + async def test_validate_cron_expression(self): + """Test cron expression validation.""" + scheduler = BackupScheduler() + + # Valid cron expressions + assert scheduler.validate_cron_expression("0 2 * * *") # Daily at 2 AM + assert scheduler.validate_cron_expression("0 */6 * * *") # Every 6 hours + assert scheduler.validate_cron_expression("0 0 * * 0") # Weekly on Sunday + + # Invalid cron expressions + assert not scheduler.validate_cron_expression("invalid") + assert not scheduler.validate_cron_expression("* * * * * *") # Too many fields + assert not scheduler.validate_cron_expression("60 0 * * *") # Invalid minute + + async def test_get_next_run_time(self): + """Test getting next run time for cron expression.""" + scheduler = BackupScheduler() + + # Test daily at 2 AM + next_run = scheduler.get_next_run_time("0 2 * * *") + assert next_run is not None + assert isinstance(next_run, datetime) + + # Should be tomorrow at 2 AM (or today if current time is before 2 AM) + assert next_run.hour == 2 + assert next_run.minute == 0 + + async def test_trigger_immediate_backup(self): + """Test triggering immediate backup.""" + with patch('app.scheduler.backup_engine') as mock_backup_engine: + mock_backup = MagicMock() + mock_backup.id = 123 + mock_backup_engine.create_backup.return_value = mock_backup + mock_backup_engine.run_backup.return_value = AsyncMock(return_value=True) + + scheduler = BackupScheduler() + + target = BackupTarget( + id=1, + name="test-volume", + target_type=TargetType.VOLUME, + source_path="test-volume", + enabled=True + ) + + backup_id = await scheduler.trigger_immediate_backup(target, BackupType.FULL) + + assert backup_id == 123 + mock_backup_engine.create_backup.assert_called_once_with(target, BackupType.FULL) + mock_backup_engine.run_backup.assert_called_once_with(123) + + async def test_scheduler_persistence_across_restarts(self): + """Test that scheduled jobs persist across scheduler restarts.""" + with patch('app.scheduler.async_session') as mock_session: + # Mock persistent schedules in database + mock_session_instance = AsyncMock() + mock_session.return_value.__aenter__.return_value = mock_session_instance + + target = BackupTarget( + id=1, + name="persistent-volume", + target_type=TargetType.VOLUME, + source_path="persistent-volume", + enabled=True + ) + + schedule = BackupSchedule( + id=1, + name="Persistent Schedule", + target_id=1, + schedule_type=ScheduleType.CRON, + cron_expression="0 3 * * *", + backup_type=BackupType.FULL, + enabled=True + ) + schedule.target = target + + mock_result = MagicMock() + mock_result.scalars.return_value.all.return_value = [schedule] + mock_session_instance.execute.return_value = mock_result + + # First scheduler instance + scheduler1 = BackupScheduler() + await scheduler1.start() + jobs1 = scheduler1.scheduler.get_jobs() + await scheduler1.stop() + + # Second scheduler instance (simulating restart) + scheduler2 = BackupScheduler() + await scheduler2.start() + jobs2 = scheduler2.scheduler.get_jobs() + await scheduler2.stop() + + # Should load same schedules from database + assert len(jobs1) == len(jobs2) + + async def test_scheduler_handles_disabled_targets(self): + """Test that scheduler doesn't schedule backups for disabled targets.""" + with patch('app.scheduler.async_session') as mock_session: + mock_session_instance = AsyncMock() + mock_session.return_value.__aenter__.return_value = mock_session_instance + + # Disabled target + target = BackupTarget( + id=1, + name="disabled-volume", + target_type=TargetType.VOLUME, + source_path="disabled-volume", + enabled=False # Disabled + ) + + schedule = BackupSchedule( + id=1, + name="Schedule for Disabled Target", + target_id=1, + schedule_type=ScheduleType.CRON, + cron_expression="0 3 * * *", + backup_type=BackupType.FULL, + enabled=True # Schedule enabled but target disabled + ) + schedule.target = target + + mock_result = MagicMock() + mock_result.scalars.return_value.all.return_value = [schedule] + mock_session_instance.execute.return_value = mock_result + + scheduler = BackupScheduler() + await scheduler.start() + + # Should not schedule job for disabled target + jobs = scheduler.scheduler.get_jobs() + assert len(jobs) == 0 + + await scheduler.stop() + + async def test_concurrent_backup_execution_limits(self): + """Test that scheduler respects concurrent backup limits.""" + scheduler = BackupScheduler() + + # Mock that maximum concurrent backups is reached + with patch('app.scheduler.backup_engine') as mock_backup_engine: + mock_backup_engine.active_backups = {1: MagicMock(), 2: MagicMock(), 3: MagicMock()} + + target = BackupTarget( + id=4, + name="test-volume", + target_type=TargetType.VOLUME, + source_path="test-volume", + enabled=True + ) + + # Should skip backup if limit reached + result = await scheduler._execute_backup(target, BackupType.FULL) + + # In real implementation, would verify backup was skipped + # due to concurrent limit + assert result is not None # Or None if skipped diff --git a/docs/TESTING_GUIDE.md b/docs/TESTING_GUIDE.md new file mode 100644 index 0000000..a8077b8 --- /dev/null +++ b/docs/TESTING_GUIDE.md @@ -0,0 +1,261 @@ +# DockerVault Testing Guide + +This guide covers all testing approaches for DockerVault to ensure backups work correctly and nothing breaks. + +## Quick Start Testing + +### 1. Run Integration Tests (Recommended First) +```bash +# From project root - tests real Docker backup/restore operations +./integration_test.sh +``` + +This will: +- Create a test Docker volume with sample data +- Perform backup operations +- Verify backup integrity +- Test restore after simulated data loss +- Test backup with running containers +- Check security (path traversal prevention) +- Test large file handling + +### 2. Run Unit Tests +```bash +# Backend Python tests +cd backend +source .venv/bin/activate # or your venv path +pytest -v + +# Frontend tests (if needed) +cd frontend +npm test +``` + +--- + +## Comprehensive Testing Checklist + +### Level 1: Smoke Tests (5 minutes) +Quick verification that the app starts and basic features work. + +- [ ] App starts without errors: `docker-compose up -d` +- [ ] Frontend loads: http://localhost:8080 +- [ ] Backend health check: `curl http://localhost:8000/api/v1/docker/health` +- [ ] Docker volumes are listed in the UI +- [ ] Docker containers are listed in the UI + +### Level 2: Functional Tests (15 minutes) +Test core backup functionality. + +#### Backup Target Creation +- [ ] Create a new volume backup target +- [ ] Create a new path backup target +- [ ] Enable/disable targets +- [ ] Delete a target + +#### Manual Backup +- [ ] Trigger a manual backup +- [ ] Monitor backup progress in real-time +- [ ] Verify backup file is created +- [ ] Check backup appears in history + +#### Backup Verification +- [ ] Download a backup file +- [ ] Verify tar.gz can be extracted +- [ ] Compare extracted content with original + +#### Restore Operation +- [ ] Restore a backup (to original or new location) +- [ ] Verify restored data matches original + +### Level 3: Stress Tests (30 minutes) +Test edge cases and limits. + +#### Large Data +- [ ] Backup a volume with 1GB+ of data +- [ ] Backup many small files (10,000+ files) +- [ ] Monitor memory usage during backup + +#### Concurrent Operations +- [ ] Run 3+ backups simultaneously +- [ ] Verify semaphore limits concurrent backups +- [ ] Check no data corruption + +#### Long-Running Containers +- [ ] Backup volume attached to actively-writing container +- [ ] Test stop-before-backup option +- [ ] Verify container restart after backup + +### Level 4: Failure/Recovery Tests (20 minutes) +Test error handling and recovery. + +- [ ] Cancel a running backup +- [ ] Disk full scenario +- [ ] Network interruption during remote storage upload +- [ ] Docker daemon restart during backup +- [ ] Invalid backup file restore attempt +- [ ] Non-existent volume/path handling + +--- + +## Manual Testing Steps + +### Test 1: Basic Volume Backup + +```bash +# 1. Create a test volume with data +docker volume create test_backup_volume +docker run --rm -v test_backup_volume:/data alpine sh -c " + echo 'Important data' > /data/important.txt + mkdir /data/subdir + echo 'Nested file' > /data/subdir/nested.txt +" + +# 2. Start DockerVault +docker-compose up -d + +# 3. Open UI at http://localhost:8080 +# 4. Go to Targets โ†’ Add Target +# 5. Select 'test_backup_volume' and save +# 6. Go to Backups โ†’ Trigger backup +# 7. Wait for completion +# 8. Verify backup file exists + +# 9. Verify backup contents +ls -la ./backups/ +tar -tzf ./backups/test_backup_volume_*.tar.gz +``` + +### Test 2: Data Integrity Verification + +```bash +# 1. Get checksum of original data +docker run --rm -v test_backup_volume:/data alpine md5sum /data/important.txt + +# 2. Extract backup and compare +mkdir /tmp/verify_backup +tar -xzf ./backups/test_backup_volume_*.tar.gz -C /tmp/verify_backup +md5sum /tmp/verify_backup/important.txt + +# Checksums should match! +``` + +### Test 3: Restore Test + +```bash +# 1. Corrupt/delete original data +docker run --rm -v test_backup_volume:/data alpine rm -rf /data/* + +# 2. Verify data is gone +docker run --rm -v test_backup_volume:/data alpine ls -la /data/ + +# 3. Use DockerVault UI to restore backup + +# 4. Verify data is restored +docker run --rm -v test_backup_volume:/data alpine cat /data/important.txt +``` + +### Test 4: Scheduled Backup + +```bash +# 1. Create a schedule (every 5 minutes for testing) +# Via UI: Schedules โ†’ Add Schedule โ†’ Cron: */5 * * * * + +# 2. Wait 5+ minutes + +# 3. Verify automatic backup was created +ls -la ./backups/ +``` + +--- + +## API Testing with curl + +```bash +# Health check +curl http://localhost:8000/api/v1/docker/health + +# List Docker volumes +curl http://localhost:8000/api/v1/docker/volumes + +# List backup targets +curl http://localhost:8000/api/v1/targets + +# List backups +curl http://localhost:8000/api/v1/backups + +# Get backup metrics +curl http://localhost:8000/api/v1/backups/metrics/summary + +# Create a backup (replace TARGET_ID with actual ID) +curl -X POST http://localhost:8000/api/v1/backups \ + -H "Content-Type: application/json" \ + -d '{"target_id": 1, "backup_type": "full"}' + +# Validate backup prerequisites +curl -X POST http://localhost:8000/api/v1/backups/1/validate +``` + +--- + +## Automated Regression Testing + +### Before Each Release + +1. **Run Unit Tests** + ```bash + cd backend && pytest --cov=app --cov-fail-under=80 + ``` + +2. **Run Integration Tests** + ```bash + ./integration_test.sh + ``` + +3. **Manual Smoke Test** + - Start fresh: `docker-compose down -v && docker-compose up -d` + - Create target, run backup, verify, restore + +4. **Check Logs for Errors** + ```bash + docker-compose logs backend | grep -i error + ``` + +--- + +## Production Safety Checklist + +Before using in production: + +- [ ] Test with your actual data volumes (on a clone/copy first!) +- [ ] Verify backup retention policy works +- [ ] Test remote storage upload (S3/FTP/WebDAV) +- [ ] Monitor disk space during scheduled backups +- [ ] Set up alerting for failed backups +- [ ] Document restore procedures +- [ ] Test restore on a different machine +- [ ] Verify backups work after DockerVault update + +--- + +## Troubleshooting + +### Backup fails with "Volume not found" +- Check volume name is correct +- Verify Docker socket permissions +- Check `docker volume ls` shows the volume + +### Backup is 0 bytes or very small +- Volume might be empty +- Check container using volume isn't holding locks +- Try stopping containers before backup + +### Restore doesn't work +- Verify backup file isn't corrupted: `tar -tzf backup.tar.gz` +- Check target path exists and is writable +- Ensure no containers are using the volume + +### Performance issues +- Reduce concurrent backup limit in settings +- Increase compression level for network storage +- Check available disk space diff --git a/frontend/package.json b/frontend/package.json index ce7773c..ca17961 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -7,7 +7,10 @@ "dev": "vite", "build": "tsc && vite build", "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", - "preview": "vite preview" + "preview": "vite preview", + "test": "vitest", + "test:coverage": "vitest --coverage", + "test:ui": "vitest --ui" }, "dependencies": { "@tanstack/react-query": "^5.17.0", @@ -25,18 +28,25 @@ "zustand": "^5.0.10" }, "devDependencies": { + "@testing-library/jest-dom": "^6.2.0", + "@testing-library/react": "^14.1.0", + "@testing-library/user-event": "^14.5.0", "@types/react": "^19.2.9", "@types/react-dom": "^19.2.0", "@typescript-eslint/eslint-plugin": "^8.53.1", "@typescript-eslint/parser": "^8.53.1", "@vitejs/plugin-react": "^4.2.1", + "@vitest/coverage-v8": "^1.2.0", "autoprefixer": "^10.4.17", "eslint": "^9.39.2", "eslint-plugin-react-hooks": "^5.0.0", "eslint-plugin-react-refresh": "^0.4.5", + "jsdom": "^23.0.0", + "msw": "^2.0.0", "postcss": "^8.4.33", "tailwindcss": "^3.4.1", "typescript": "^5.3.3", - "vite": "^7.3.1" + "vite": "^7.3.1", + "vitest": "^1.2.0" } } diff --git a/frontend/src/api/__tests__/index.test.ts b/frontend/src/api/__tests__/index.test.ts new file mode 100644 index 0000000..150777f --- /dev/null +++ b/frontend/src/api/__tests__/index.test.ts @@ -0,0 +1,367 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest' +import { server } from '../../test/mocks/server' +import { http, HttpResponse } from 'msw' +import { backupsApi, dockerApi, targetsApi, schedulesApi } from '../index' + +describe('API Layer', () => { + beforeEach(() => { + // Reset any mocks between tests + vi.clearAllMocks() + }) + + describe('backupsApi', () => { + it('should fetch backups list', async () => { + const response = await backupsApi.list() + + expect(response.data).toHaveLength(2) + expect(response.data[0]).toMatchObject({ + id: 1, + target_name: 'test-volume', + backup_type: 'full', + status: 'completed', + }) + }) + + it('should fetch single backup by id', async () => { + const response = await backupsApi.get(1) + + expect(response.data).toMatchObject({ + id: 1, + target_name: 'test-volume', + backup_type: 'full', + status: 'completed', + }) + }) + + it('should handle backup not found', async () => { + await expect(backupsApi.get(99999)).rejects.toThrow() + }) + + it('should create new backup', async () => { + const response = await backupsApi.create(1, 'full') + + expect(response.data).toMatchObject({ + target_id: 1, + backup_type: 'full', + status: 'pending', + }) + }) + + it('should handle create backup errors', async () => { + await expect(backupsApi.create(99999, 'full')).rejects.toThrow() + }) + + it('should delete backup', async () => { + const response = await backupsApi.delete(1) + expect(response.status).toBe(204) + }) + + it('should restore backup', async () => { + const response = await backupsApi.restore(1) + expect(response.data).toMatchObject({ + message: 'Restore initiated', + }) + }) + + it('should list backups with filters', async () => { + server.use( + http.get('/api/v1/backups', ({ request }) => { + const url = new URL(request.url) + const targetId = url.searchParams.get('target_id') + const status = url.searchParams.get('status') + + expect(targetId).toBe('1') + expect(status).toBe('completed') + + return HttpResponse.json([]) + }) + ) + + await backupsApi.list({ target_id: 1, status: 'completed' }) + }) + }) + + describe('dockerApi', () => { + it('should list containers', async () => { + const response = await dockerApi.listContainers() + + expect(response.data).toHaveLength(1) + expect(response.data[0]).toMatchObject({ + id: 'container123', + name: 'test-container', + image: 'nginx:latest', + status: 'running', + }) + }) + + it('should list volumes', async () => { + const response = await dockerApi.listVolumes() + + expect(response.data).toHaveLength(1) + expect(response.data[0]).toMatchObject({ + name: 'test-volume', + driver: 'local', + used_by: ['test-container'], + }) + }) + + it('should handle Docker daemon unavailability', async () => { + server.use( + http.get('/api/v1/docker/containers', () => { + return new HttpResponse(null, { + status: 503, + statusText: 'Docker daemon unavailable', + }) + }) + ) + + await expect(dockerApi.listContainers()).rejects.toThrow() + }) + + it('should stop container', async () => { + server.use( + http.post('/api/v1/docker/containers/:id/stop', ({ params }) => { + expect(params.id).toBe('container123') + return HttpResponse.json({ message: 'Container stopped' }) + }) + ) + + const response = await dockerApi.stopContainer('container123') + expect(response.data.message).toBe('Container stopped') + }) + + it('should start container', async () => { + server.use( + http.post('/api/v1/docker/containers/:id/start', ({ params }) => { + expect(params.id).toBe('container123') + return HttpResponse.json({ message: 'Container started' }) + }) + ) + + const response = await dockerApi.startContainer('container123') + expect(response.data.message).toBe('Container started') + }) + }) + + describe('targetsApi', () => { + it('should list targets', async () => { + const response = await targetsApi.list() + + expect(response.data).toHaveLength(1) + expect(response.data[0]).toMatchObject({ + id: 1, + name: 'test-volume', + target_type: 'volume', + enabled: true, + }) + }) + + it('should get single target', async () => { + const response = await targetsApi.get(1) + + expect(response.data).toMatchObject({ + id: 1, + name: 'test-volume', + target_type: 'volume', + enabled: true, + }) + }) + + it('should create new target', async () => { + server.use( + http.post('/api/v1/targets', async ({ request }) => { + const body = await request.json() + return HttpResponse.json({ + id: Date.now(), + ...body, + created_at: new Date().toISOString(), + }, { status: 201 }) + }) + ) + + const targetData = { + name: 'new-volume', + target_type: 'volume' as const, + source_path: 'new-volume', + enabled: true, + } + + const response = await targetsApi.create(targetData) + expect(response.data).toMatchObject(targetData) + }) + + it('should update target', async () => { + server.use( + http.put('/api/v1/targets/:id', async ({ params, request }) => { + const body = await request.json() + expect(params.id).toBe('1') + return HttpResponse.json({ + id: 1, + ...body, + updated_at: new Date().toISOString(), + }) + }) + ) + + const updates = { enabled: false } + const response = await targetsApi.update(1, updates) + expect(response.data.enabled).toBe(false) + }) + + it('should delete target', async () => { + server.use( + http.delete('/api/v1/targets/:id', ({ params }) => { + expect(params.id).toBe('1') + return new HttpResponse(null, { status: 204 }) + }) + ) + + const response = await targetsApi.delete(1) + expect(response.status).toBe(204) + }) + }) + + describe('schedulesApi', () => { + it('should list schedules', async () => { + server.use( + http.get('/api/v1/schedules', () => { + return HttpResponse.json([ + { + id: 1, + target_id: 1, + target_name: 'test-volume', + cron_expression: '0 2 * * *', + enabled: true, + next_run: '2024-01-02T02:00:00Z', + }, + ]) + }) + ) + + const response = await schedulesApi.list() + expect(response.data).toHaveLength(1) + expect(response.data[0].cron_expression).toBe('0 2 * * *') + }) + + it('should trigger manual backup', async () => { + server.use( + http.post('/api/v1/schedules/:targetId/trigger', ({ params }) => { + expect(params.targetId).toBe('1') + return HttpResponse.json({ message: 'Backup triggered' }) + }) + ) + + const response = await schedulesApi.trigger(1) + expect(response.data.message).toBe('Backup triggered') + }) + + it('should validate cron expressions', async () => { + server.use( + http.post('/api/v1/schedules/estimate', async ({ request }) => { + const body = await request.json() as { target_id: number, cron_expression: string } + + if (body.cron_expression === 'invalid') { + return new HttpResponse(JSON.stringify({ detail: 'Invalid cron expression' }), { + status: 400, + headers: { 'Content-Type': 'application/json' }, + }) + } + + return HttpResponse.json({ + next_runs: ['2024-01-02T02:00:00Z', '2024-01-03T02:00:00Z'], + description: 'Daily at 2:00 AM', + }) + }) + ) + + const response = await schedulesApi.estimate(1, '0 2 * * *') + expect(response.data.description).toBe('Daily at 2:00 AM') + + await expect( + schedulesApi.estimate(1, 'invalid') + ).rejects.toThrow() + }) + }) + + describe('Error Handling', () => { + it('should handle network errors', async () => { + server.use( + http.get('/api/v1/backups', () => { + return HttpResponse.error() + }) + ) + + await expect(backupsApi.list()).rejects.toThrow() + }) + + it('should handle 500 server errors', async () => { + server.use( + http.get('/api/v1/backups', () => { + return new HttpResponse(null, { + status: 500, + statusText: 'Internal Server Error', + }) + }) + ) + + await expect(backupsApi.list()).rejects.toThrow() + }) + + it('should handle JSON parsing errors', async () => { + server.use( + http.get('/api/v1/backups', () => { + return new HttpResponse('invalid json', { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + }) + ) + + await expect(backupsApi.list()).rejects.toThrow() + }) + }) + + describe('Security', () => { + it('should include credentials in requests', async () => { + server.use( + http.get('/api/v1/backups', ({ request }) => { + // In real implementation, check for credentials/cookies + expect(request.credentials).toBe('include') + return HttpResponse.json([]) + }) + ) + + await backupsApi.list() + }) + + it('should sanitize input parameters', async () => { + server.use( + http.get('/api/v1/backups', ({ request }) => { + const url = new URL(request.url) + const targetId = url.searchParams.get('target_id') + + // Should not contain script tags or SQL injection attempts + expect(targetId).not.toContain('