From 84fee602839edc64b9288b472c707eda2781c053 Mon Sep 17 00:00:00 2001 From: Jannik Reinhard Date: Mon, 27 Jul 2026 07:25:10 +0200 Subject: [PATCH] fix runtime and consolidate document manager --- .dockerignore | 25 ++++++++ .env.example | 50 +++++++++++++++ .github/workflows/ci.yml | 2 +- .gitignore | 35 +++++++---- Dockerfile | 19 ++---- Dockerfile.dev | 14 ++--- README.md | 62 +++++++++++------- app/config.py | 1 + app/routers/backup.py | 4 +- app/services/backup_scheduler.py | 6 +- app/utils/backup.py | 44 ++++++------- app/utils/file_security.py | 4 +- docker-entrypoint-aio.sh | 69 -------------------- docker-entrypoint.sh | 20 +++++- requirements.txt | 6 +- setup.sh | 97 +++++++++++++---------------- supervisord.conf | 38 ----------- tests/test_backup_security.py | 73 ++++++++++++++++++++++ tests/test_runtime_configuration.py | 55 ++++++++++++++++ 19 files changed, 365 insertions(+), 259 deletions(-) create mode 100644 .dockerignore create mode 100644 .env.example delete mode 100755 docker-entrypoint-aio.sh delete mode 100644 supervisord.conf create mode 100644 tests/test_backup_security.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e1b7f93 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,25 @@ +.git +.github +.env +.env.* +!.env.example +.venv +venv +__pycache__ +*.py[cod] +.pytest_cache + +backups +chroma +data +logs +staging +storage +uploads + +assets +tests +README.md +Dockerfile.dev +setup.ps1 +setup.sh diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..5df4dbd --- /dev/null +++ b/.env.example @@ -0,0 +1,50 @@ +# Copy this file to .env. Never commit real credentials. + +# Security +SECRET_KEY= +JWT_SECRET_KEY= +ALGORITHM=HS256 +ACCESS_TOKEN_EXPIRE_MINUTES=30 +ENVIRONMENT=production +CORS_ORIGINS=http://localhost:8000,http://127.0.0.1:8000 +TRUSTED_PROXY_IPS=127.0.0.1,::1 + +# Database and storage +DATABASE_URL=sqlite:///./data/documents.db +DATA_FOLDER=./data +STAGING_FOLDER=./data/staging +STORAGE_FOLDER=./data/storage +LOGS_FOLDER=./data/logs +BACKUP_FOLDER=./data/backups + +# AI provider: openai or azure +AI_PROVIDER=openai +OPENAI_API_KEY= +EMBEDDING_MODEL=text-embedding-3-small +CHAT_MODEL=gpt-4o-mini +ANALYSIS_MODEL=gpt-4o-mini + +# Azure OpenAI +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_ENDPOINT= +AZURE_OPENAI_API_VERSION=2024-08-01-preview +AZURE_OPENAI_CHAT_DEPLOYMENT= +AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT= + +# ChromaDB. The localhost defaults use the embedded persistent client. +CHROMA_HOST=localhost +CHROMA_PORT=8001 +CHROMA_COLLECTION_NAME=documents + +# OCR and file processing +TESSERACT_PATH=/usr/bin/tesseract +POPPLER_PATH=/usr/bin +MAX_FILE_SIZE=100MB +ALLOWED_EXTENSIONS=pdf,png,jpg,jpeg,tiff,bmp,txt,text,md,markdown + +# Runtime behavior +LOG_LEVEL=INFO +AI_TEXT_LIMIT=16000 +AI_CONTEXT_LIMIT=10000 +AI_REQUEST_TIMEOUT=30 +AI_MAX_RETRIES=2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f2282a..161a899 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: python-version: "3.12" cache: pip - name: Install test dependencies - run: pip install fastapi pydantic-settings sqlalchemy loguru watchdog pytest + run: pip install fastapi pydantic-settings sqlalchemy loguru watchdog passlib pytesseract Pillow pytest - name: Run regression tests run: python -m pytest tests -q - name: Parse PowerShell scripts diff --git a/.gitignore b/.gitignore index 56fde4c..2e4d172 100644 --- a/.gitignore +++ b/.gitignore @@ -1,17 +1,26 @@ -chroma/chroma.sqlite3 -data/documents.db -data/documents.db-* -data/.local-smoke-credentials.json -data/chroma/ -data/logs/ -data/staging/ -data/storage/ -data/uploads/ -data/backups/ -__pycache__/ -*.py[cod] -.DS_Store +# Local credentials and environments +.env +.venv/ +venv/ + +# Runtime data +/backups/ +/chroma/ +/data/ +/logs/ +/staging/ +/storage/ +/uploads/ +.local-smoke-credentials.json + # Python bytecode and test caches __pycache__/ *.py[cod] .pytest_cache/ +.coverage +htmlcov/ + +# Editors and operating systems +.DS_Store +.idea/ +.vscode/ diff --git a/Dockerfile b/Dockerfile index 43452f6..7ae8ec0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,8 +45,6 @@ RUN apt-get update && apt-get install -y \ libmagic1 \ # Health check curl \ - # Process management for all-in-one mode - supervisor \ # Clean up && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -66,15 +64,11 @@ WORKDIR /app COPY --chown=appuser:appuser app/ ./app/ COPY --chown=appuser:appuser frontend/ ./frontend/ COPY --chown=appuser:appuser docker-entrypoint.sh ./ -COPY --chown=appuser:appuser docker-entrypoint-aio.sh ./ -COPY --chown=appuser:appuser supervisord.conf /etc/supervisor/conf.d/ # Create necessary directories with correct permissions -RUN mkdir -p data data/logs data/staging data/storage data/uploads backups chroma && \ - chmod +x docker-entrypoint.sh docker-entrypoint-aio.sh && \ - chown -R appuser:appuser /app && \ - mkdir -p /var/log/supervisor && \ - chown -R appuser:appuser /var/log/supervisor +RUN mkdir -p data/logs data/staging data/storage data/uploads data/backups && \ + chmod +x docker-entrypoint.sh && \ + chown -R appuser:appuser /app # Switch to non-root user USER appuser @@ -86,17 +80,16 @@ ENV PYTHONUNBUFFERED=1 \ # Tesseract and Poppler paths TESSERACT_PATH=/usr/bin/tesseract \ POPPLER_PATH=/usr/bin \ - # Application settings (should be overridden in production) + # Application settings DATABASE_URL=sqlite:///./data/documents.db \ - SECRET_KEY=MUST-BE-SET-IN-PRODUCTION \ AI_PROVIDER=openai \ TRUSTED_PROXY_IPS=127.0.0.1 # Add metadata labels -LABEL maintainer="Document Manager Team" \ +LABEL maintainer="Jannik Reinhard" \ version="1.0.0" \ description="AI-powered document management system" \ - org.opencontainers.image.source="https://github.com/yourusername/documentmanager" + org.opencontainers.image.source="https://github.com/JayRHa/DocumentManager" # Expose application port EXPOSE 8000 diff --git a/Dockerfile.dev b/Dockerfile.dev index 6ae1009..d63c916 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -17,8 +17,6 @@ RUN apt-get update && apt-get install -y \ libmagic1 libmagic-dev \ # Health check curl \ - # Process management - supervisor \ # Development tools vim nano htop \ && apt-get clean \ @@ -29,20 +27,17 @@ WORKDIR /app # Copy requirements and install Python packages COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt && \ - pip install --no-cache-dir chromadb watchdog +RUN pip install --no-cache-dir -r requirements.txt # Copy application code COPY app/ ./app/ COPY frontend/ ./frontend/ COPY cli.py ./ COPY docker-entrypoint.sh ./ -COPY docker-entrypoint-aio.sh ./ -COPY supervisord.conf /etc/supervisor/conf.d/ # Create necessary directories -RUN mkdir -p data data/logs data/staging data/storage data/uploads backups chroma /var/log/supervisor && \ - chmod +x docker-entrypoint.sh docker-entrypoint-aio.sh +RUN mkdir -p data/logs data/staging data/storage data/uploads data/backups && \ + chmod +x docker-entrypoint.sh # Development environment variables ENV PYTHONUNBUFFERED=1 \ @@ -51,7 +46,6 @@ ENV PYTHONUNBUFFERED=1 \ ENVIRONMENT=development \ LOG_LEVEL=DEBUG \ DATABASE_URL=sqlite:///./data/documents.db \ - SECRET_KEY=dev-secret-key-only-for-development \ TESSERACT_PATH=/usr/bin/tesseract \ POPPLER_PATH=/usr/bin @@ -59,4 +53,4 @@ ENV PYTHONUNBUFFERED=1 \ EXPOSE 8000 # Use the same entrypoint as production -ENTRYPOINT ["/app/docker-entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["/app/docker-entrypoint.sh"] diff --git a/README.md b/README.md index 84a388e..271e261 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,10 @@ +> **Canonical repository:** This repository supersedes the legacy +> `JayRHa/DocumentManagement` project and contains the maintained application, +> runtime configuration, tests, and deployment path. + ## Features ### AI-Powered Intelligence @@ -93,15 +97,20 @@ The beauty of open source? You can have this running on your machine right now: git clone https://github.com/JayRHa/DocumentManager.git cd DocumentManager -# Run the setup script +# Build and run with a generated local .env file +./setup.sh build ./setup.sh prod # Or manually with Docker docker build -t documentmanager:local . +cp .env.example .env +# Set a strong SECRET_KEY and optional AI credentials in .env first. docker run -d \ --name documentmanager-local \ -p 127.0.0.1:8000:8000 \ + --env-file .env \ -v $(pwd)/data:/app/data \ + -v $(pwd)/backups:/app/data/backups \ documentmanager:local ``` @@ -205,15 +214,16 @@ uvicorn app.main:app --reload --host 127.0.0.1 --port 8000 ``` DocumentManager/ -├── app/ # Backend FastAPI application -│ ├── api/ # REST API endpoints -│ ├── core/ # Core business logic -│ ├── models/ # SQLAlchemy models -│ └── services/ # AI, OCR, and storage services -├── frontend/ # Vanilla JS frontend -├── docker/ # Docker configuration -├── tests/ # Test suite -└── docs/ # Documentation +├── app/ # FastAPI backend +│ ├── middleware/ # Authentication, CSRF, rate limiting, logging +│ ├── routers/ # REST API endpoints +│ ├── services/ # AI, OCR, search, and document processing +│ └── utils/ # Backup, validation, and file security +├── frontend/ # Vanilla JavaScript frontend +├── tests/ # Regression tests +├── Dockerfile # Production container +├── Dockerfile.dev # Development container +└── setup.sh / setup.ps1 # Runtime helpers ``` ### Technology Stack @@ -221,23 +231,30 @@ DocumentManager/ - **Backend**: FastAPI, SQLAlchemy, Pydantic - **AI/ML**: OpenAI GPT-4, Azure OpenAI, ChromaDB - **OCR**: Tesseract (50+ languages) -- **Database**: SQLite (default), PostgreSQL (production) +- **Database**: SQLite - **Frontend**: Vanilla JavaScript, modern CSS -- **Deployment**: Docker, Docker Compose +- **Deployment**: Docker or Podman ## Configuration ### Environment Variables -Create a `.env` file in the root directory: +Copy `.env.example` to `.env`. The setup script does this automatically and +generates a strong `SECRET_KEY` when `.env` does not exist. ```bash -# Security - CHANGE IN PRODUCTION! -SECRET_KEY=your-secret-key-here +cp .env.example .env +python -c 'import secrets; print(secrets.token_urlsafe(32))' +``` + +Place the generated value in `SECRET_KEY` and configure the required provider: + +```dotenv +SECRET_KEY=replace-with-generated-value +ENVIRONMENT=production # Database DATABASE_URL=sqlite:///./data/documents.db -# For PostgreSQL: postgresql://user:pass@localhost/dbname # AI Provider AI_PROVIDER=openai @@ -245,17 +262,14 @@ OPENAI_API_KEY=sk-... # Or for Azure: # AI_PROVIDER=azure # AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com -# AZURE_OPENAI_KEY=your-key +# AZURE_OPENAI_API_KEY=your-key +# AZURE_OPENAI_CHAT_DEPLOYMENT=your-chat-deployment +# AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT=your-embedding-deployment # Application Settings -ENVIRONMENT=production LOG_LEVEL=INFO -MAX_UPLOAD_SIZE=104857600 # 100MB -ALLOWED_EXTENSIONS=pdf,jpg,jpeg,png,txt,md,markdown,doc,docx - -# Storage -STORAGE_TYPE=local -STORAGE_PATH=/app/data/storage +MAX_FILE_SIZE=100MB +ALLOWED_EXTENSIONS=pdf,png,jpg,jpeg,tiff,bmp,txt,text,md,markdown ``` ## API Documentation diff --git a/app/config.py b/app/config.py index a5815b8..4f3594b 100644 --- a/app/config.py +++ b/app/config.py @@ -35,6 +35,7 @@ class Settings(BaseSettings): data_folder: str = "./data" storage_folder: str = "./data/storage" logs_folder: str = "./data/logs" + backup_folder: str = "./data/backups" # OCR tesseract_path: str = "/usr/bin/tesseract" diff --git a/app/routers/backup.py b/app/routers/backup.py index caa97b6..9016dff 100644 --- a/app/routers/backup.py +++ b/app/routers/backup.py @@ -48,7 +48,7 @@ class BackupConfigRequest(BaseModel): interval_hours: int = 24 max_backups: int = 7 include_files: bool = True - backup_path: str = "backups" + backup_path: str = "data/backups" class ManualBackupRequest(BaseModel): @@ -412,4 +412,4 @@ def backup_health_check( "message": f"Health check failed: {str(e)}", "recommendation": "Check backup system configuration" }] - } \ No newline at end of file + } diff --git a/app/services/backup_scheduler.py b/app/services/backup_scheduler.py index 1529771..0114ad6 100644 --- a/app/services/backup_scheduler.py +++ b/app/services/backup_scheduler.py @@ -27,7 +27,7 @@ def __init__(self): "interval_hours": 24, # Default: daily backups "max_backups": 7, # Keep last 7 backups "include_files": True, - "backup_path": "backups" + "backup_path": "data/backups" } self.last_backup: Optional[datetime] = None self.backup_history = [] @@ -38,7 +38,7 @@ def configure( interval_hours: int = 24, max_backups: int = 7, include_files: bool = True, - backup_path: str = "backups" + backup_path: str = "data/backups" ): """ Configure backup scheduler settings. @@ -384,4 +384,4 @@ def get_backup_recommendations(self) -> Dict[str, Any]: # Global backup scheduler instance -backup_scheduler = BackupScheduler() \ No newline at end of file +backup_scheduler = BackupScheduler() diff --git a/app/utils/backup.py b/app/utils/backup.py index 582eeae..6446fee 100644 --- a/app/utils/backup.py +++ b/app/utils/backup.py @@ -13,7 +13,7 @@ from loguru import logger from ..database import engine -from ..config import get_settings +from ..config import Settings, get_settings from ..models import User @@ -28,8 +28,9 @@ def _extract_tar_safely(tar: tarfile.TarFile, dest: Path) -> None: Older Python releases (< 3.12) don't support the ``filter='data'`` argument to :py:meth:`tarfile.TarFile.extractall`, so we manually validate each member's destination against the target directory. - Members whose resolved path escapes ``dest``, absolute paths, and - symlinks/hard-links that point outside ``dest`` are rejected. + Members whose resolved path escapes ``dest``, links, and special files are + rejected. Backup archives only need regular files and directories, so + accepting links would add an unnecessary extraction attack surface. See CVE-2007-4559 / PEP 706 for background. """ @@ -44,16 +45,9 @@ def _extract_tar_safely(tar: tarfile.TarFile, dest: Path) -> None: f"Refusing to extract tar member outside dest: {member.name!r}" ) if member.issym() or member.islnk(): - link_target = (member_path.parent / member.linkname).resolve() - try: - link_target.relative_to(dest_root) - except ValueError: - raise BackupError( - f"Refusing to extract tar link outside dest: {member.name!r}" - f" -> {member.linkname!r}" - ) - elif not (member.isfile() or member.isdir()): - raise BackupError(f"Refusing to extract unsupported tar member: {member.name!r}") + raise BackupError(f"Refusing to extract tar link: {member.name!r}") + if not (member.isfile() or member.isdir()): + raise BackupError(f"Refusing to extract special tar member: {member.name!r}") for member in members: tar.extract(member, dest) @@ -84,8 +78,8 @@ def create_backup( backup_name = backup_name or f"backup_{timestamp}" # Create backup directory - backup_base = Path(settings.get('backup_path', 'data/backups')) - backup_base.mkdir(exist_ok=True) + backup_base = Path(settings.backup_folder) + backup_base.mkdir(parents=True, exist_ok=True) backup_dir = backup_base / backup_name backup_dir.mkdir(exist_ok=True) @@ -119,8 +113,8 @@ def create_backup( 'database_engine': str(engine.url).split('://')[0], 'include_files': include_files, 'settings': { - 'storage_folder': settings.get('storage_folder'), - 'staging_folder': settings.get('staging_folder'), + 'storage_folder': settings.storage_folder, + 'staging_folder': settings.staging_folder, }, 'statistics': get_backup_statistics(db_session) } @@ -239,7 +233,7 @@ def backup_database(backup_dir: Path) -> Path: raise BackupError(f"Unsupported database type: {db_url.split('://')[0]}") -def backup_files(backup_dir: Path, settings: Dict[str, Any]) -> Path: +def backup_files(backup_dir: Path, settings: Settings) -> Path: """ Backup document files to the specified directory. @@ -251,17 +245,17 @@ def backup_files(backup_dir: Path, settings: Dict[str, Any]) -> Path: Path to the files backup """ files_backup_dir = backup_dir / 'files' - files_backup_dir.mkdir(exist_ok=True) + files_backup_dir.mkdir(parents=True, exist_ok=True) # Backup storage folder - storage_path = Path(settings.get('storage_folder', 'data/storage')) + storage_path = Path(settings.storage_folder) if storage_path.exists(): storage_backup = files_backup_dir / 'storage' shutil.copytree(storage_path, storage_backup, dirs_exist_ok=True) logger.info(f"Backed up storage folder: {storage_path}") # Backup staging folder (optional) - staging_path = Path(settings.get('staging_folder', 'data/staging')) + staging_path = Path(settings.staging_folder) if staging_path.exists(): staging_backup = files_backup_dir / 'staging' shutil.copytree(staging_path, staging_backup, dirs_exist_ok=True) @@ -323,8 +317,8 @@ def restore_backup( temp_path = Path(temp_dir) try: - # Extract archive through the same validation path on every - # supported Python version to avoid CVE-2007-4559 style traversal. + # Use the same strict validation on every supported Python version + # to block CVE-2007-4559 style traversal and link attacks. logger.info(f"Extracting backup archive: {archive_path}") with tarfile.open(archive_path, "r:gz") as tar: _extract_tar_safely(tar, temp_path) @@ -448,7 +442,7 @@ def restore_files_from_backup(backup_dir: Path, db_session): # Restore storage folder storage_backup = files_backup_dir / 'storage' if storage_backup.exists(): - storage_path = Path(settings.get('storage_folder', 'data/storage')) + storage_path = Path(settings.storage_folder) # Backup current files if storage_path.exists(): @@ -464,7 +458,7 @@ def restore_files_from_backup(backup_dir: Path, db_session): # Restore staging folder staging_backup = files_backup_dir / 'staging' if staging_backup.exists(): - staging_path = Path(settings.get('staging_folder', 'data/staging')) + staging_path = Path(settings.staging_folder) # Clear current staging if staging_path.exists(): diff --git a/app/utils/file_security.py b/app/utils/file_security.py index 9e4ee1a..e2b6e8e 100644 --- a/app/utils/file_security.py +++ b/app/utils/file_security.py @@ -87,8 +87,8 @@ def check_file_permissions(file_path: Path, user: User) -> bool: # Check if file is in allowed directories settings = get_settings() allowed_dirs = [ - Path(settings.get('storage_path', 'data/storage')), - Path(settings.get('staging_path', 'data/staging')) + Path(settings.storage_folder), + Path(settings.staging_folder), ] try: diff --git a/docker-entrypoint-aio.sh b/docker-entrypoint-aio.sh deleted file mode 100755 index ae61cc2..0000000 --- a/docker-entrypoint-aio.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# All-in-One Docker entrypoint for Document Manager with embedded ChromaDB -set -e - -echo "🚀 Starting All-in-One Document Manager..." -echo "📅 $(date)" -echo "🔧 Environment: ${ENVIRONMENT:-production}" - -# Set OCR tool paths for Docker environment -export TESSERACT_PATH="/usr/bin/tesseract" -export POPPLER_PATH="/usr/bin" - -# Create directories if needed -echo "📁 Creating directories..." -mkdir -p /app/data /app/data/logs /app/data/staging /app/data/storage /app/data/uploads /app/data/backups /app/data/chroma - -# Check required environment variables for production -if [ "$ENVIRONMENT" = "production" ]; then - if [ -z "$SECRET_KEY" ] || [ "$SECRET_KEY" = "MUST-BE-SET-IN-PRODUCTION" ]; then - echo "⚠️ ERROR: SECRET_KEY must be set in production!" - echo "Generate a secure key with: python -c 'import secrets; print(secrets.token_urlsafe(32))'" - exit 1 - fi -fi - -# Initialize database if needed -if [ ! -f "/app/data/documents.db" ]; then - echo "🔧 Initializing database..." - python -c " -from app.database import engine, Base -from app.models import * -print('Creating database tables...') -Base.metadata.create_all(bind=engine) -print('✅ Database initialized') -" -fi - -# Check if any users exist -python -c " -from app.database import get_db -from app.models import User - -db = next(get_db()) -user_count = db.query(User).count() - -if user_count == 0: - print('ℹ️ No users found in database.') - print(' Please create an administrator account through the web interface.') -else: - print('✅ Found {} existing user(s) in database'.format(user_count)) -db.close() -" || echo "⚠️ Warning: Could not check user count" - -# Wait for ChromaDB to be ready (supervisor will start it) -echo "⏳ Starting services with supervisor..." - -# Check if we're in development mode -if [ "$ENVIRONMENT" = "development" ]; then - echo "🔧 Development mode detected - enabling hot reload" - # Copy supervisor config to writable location - cp /etc/supervisor/conf.d/supervisord.conf /tmp/supervisord.conf - # Update the copy to use reload flag - sed -i 's|--port 8000|--port 8000 --reload|' /tmp/supervisord.conf - # Use the modified config - exec /usr/bin/supervisord -c /tmp/supervisord.conf -else - # Start supervisor with original config - exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf -fi \ No newline at end of file diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index e3cb7dc..670ec7a 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -56,6 +56,20 @@ db.close() echo "✅ Initialization complete" -# Always run in all-in-one mode with embedded ChromaDB -echo "🔄 Starting in all-in-one mode with embedded ChromaDB..." -exec /app/docker-entrypoint-aio.sh \ No newline at end of file +# ChromaDB is opened directly through its persistent client. Running a second +# Chroma server in this container would duplicate storage and waste resources. +echo "🔄 Starting Document Manager with embedded persistent ChromaDB..." + +uvicorn_args=( + app.main:app + --host 0.0.0.0 + --port 8000 + --proxy-headers + --forwarded-allow-ips "${TRUSTED_PROXY_IPS:-127.0.0.1}" +) + +if [ "${ENVIRONMENT:-production}" = "development" ]; then + uvicorn_args+=(--reload) +fi + +exec python -m uvicorn "${uvicorn_args[@]}" diff --git a/requirements.txt b/requirements.txt index 2f74443..a13647e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,8 +8,8 @@ pytesseract==0.3.10 Pillow==12.2.0 python-multipart==0.0.28 watchdog==4.0.0 -pydantic==2.5.3 -pydantic-settings==2.1.0 +pydantic==2.12.5 +pydantic-settings==2.12.0 bcrypt==4.1.2 passlib[bcrypt]==1.7.4 python-jose[cryptography]==3.5.0 @@ -21,7 +21,7 @@ python-magic==0.4.27 numpy==1.26.4 loguru==0.7.2 pytest==9.0.3 -pytest-asyncio==0.23.5 +pytest-asyncio==1.3.0 httpx==0.26.0 tabulate==0.9.0 bleach==6.1.0 diff --git a/setup.sh b/setup.sh index 649295b..c06c239 100755 --- a/setup.sh +++ b/setup.sh @@ -60,51 +60,47 @@ check_requirements() { # Create .env file if it doesn't exist create_env_file() { if [ ! -f .env ]; then - print_info "Creating .env file..." - cat > .env << EOF -# Security - CHANGE THIS IN PRODUCTION! -SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe(32))' 2>/dev/null || echo "change-me-in-production") - -# Database -DATABASE_URL=sqlite:///./data/documents.db - -# AI Provider (optional) -AI_PROVIDER=openai -# OPENAI_API_KEY=your-key-here - -# Application -ENVIRONMENT=production -LOG_LEVEL=INFO -EOF + print_info "Creating .env file from .env.example..." + umask 077 + cp .env.example .env + generated_secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe(32))') + awk -v secret="$generated_secret" ' + BEGIN { replaced = 0 } + /^SECRET_KEY=$/ && !replaced { + print "SECRET_KEY=" secret + replaced = 1 + next + } + { print } + ' .env > .env.tmp + mv .env.tmp .env + chmod 600 .env print_warn "Created .env file. Please update with your settings!" fi } -# Load environment variables -load_env() { - if [ -f .env ]; then - export $(cat .env | grep -v '^#' | xargs) - fi +container_exists() { + $RUNTIME_CMD container inspect "$1" >/dev/null 2>&1 } # Development mode start_dev() { print_info "Building development image..." $RUNTIME_CMD build -f Dockerfile.dev -t ${IMAGE_NAME}:dev . - + + if container_exists "${CONTAINER_NAME}-dev"; then + print_error "Container ${CONTAINER_NAME}-dev already exists. Run './setup.sh stop' first." + exit 1 + fi + print_info "Starting development environment..." $RUNTIME_CMD run -d \ --name ${CONTAINER_NAME}-dev \ -p 8000:8000 \ - -v $(pwd)/app:/app/app:z \ - -v $(pwd)/frontend:/app/frontend:z \ - -v $(pwd)/data:/app/data:z \ - -v $(pwd)/staging:/app/staging:z \ - -v $(pwd)/storage:/app/storage:z \ - -v $(pwd)/uploads:/app/uploads:z \ - -v $(pwd)/logs:/app/logs:z \ - -v $(pwd)/backups:/app/backups:z \ - -v $(pwd)/chroma:/app/chroma:z \ + -v "$(pwd)/app:/app/app:z" \ + -v "$(pwd)/frontend:/app/frontend:z" \ + -v "$(pwd)/data:/app/data:z" \ + -v "$(pwd)/backups:/app/data/backups:z" \ ${IMAGE_NAME}:dev print_info "Development environment started!" @@ -115,36 +111,31 @@ start_dev() { start_prod() { print_info "Starting production environment..." create_env_file - load_env - - # Check if SECRET_KEY is still default - if [ "$SECRET_KEY" = "change-me-in-production" ]; then - print_warn "Using default SECRET_KEY. Please change it for production!" + + secret_key=$(sed -n 's/^SECRET_KEY=//p' .env | tail -n 1) + if [ -z "$secret_key" ] || [ "$secret_key" = "change-me-in-production" ] || [ "$secret_key" = "MUST-BE-SET-IN-PRODUCTION" ]; then + print_error "SECRET_KEY must be set to a non-default value in .env." + exit 1 fi # Always use local build for testing IMAGE="${IMAGE_NAME}:latest" - if ! $RUNTIME_CMD image exists ${IMAGE} &>/dev/null; then + if ! $RUNTIME_CMD image inspect "$IMAGE" >/dev/null 2>&1; then print_error "Local image not found. Run './setup.sh build' first." exit 1 fi + + if container_exists "${CONTAINER_NAME}"; then + print_error "Container ${CONTAINER_NAME} already exists. Run './setup.sh stop' first." + exit 1 + fi $RUNTIME_CMD run -d \ --name ${CONTAINER_NAME} \ -p 8000:8000 \ - -e SECRET_KEY="${SECRET_KEY}" \ - -e DATABASE_URL="${DATABASE_URL}" \ - -e AI_PROVIDER="${AI_PROVIDER}" \ - -e OPENAI_API_KEY="${OPENAI_API_KEY:-}" \ - -e ENVIRONMENT="${ENVIRONMENT}" \ - -e LOG_LEVEL="${LOG_LEVEL}" \ - -v $(pwd)/data:/app/data:z \ - -v $(pwd)/staging:/app/staging:z \ - -v $(pwd)/storage:/app/storage:z \ - -v $(pwd)/uploads:/app/uploads:z \ - -v $(pwd)/logs:/app/logs:z \ - -v $(pwd)/backups:/app/backups:z \ - -v $(pwd)/chroma:/app/chroma:z \ + --env-file .env \ + -v "$(pwd)/data:/app/data:z" \ + -v "$(pwd)/backups:/app/data/backups:z" \ --restart unless-stopped \ $IMAGE @@ -164,14 +155,14 @@ stop_containers() { print_info "Stopping containers..." # Stop dev container if running - if $RUNTIME_CMD ps -a | grep -q ${CONTAINER_NAME}-dev; then + if container_exists "${CONTAINER_NAME}-dev"; then $RUNTIME_CMD stop ${CONTAINER_NAME}-dev $RUNTIME_CMD rm ${CONTAINER_NAME}-dev print_info "Development container stopped and removed." fi # Stop prod container if running - if $RUNTIME_CMD ps -a | grep -q ${CONTAINER_NAME}; then + if container_exists "${CONTAINER_NAME}"; then $RUNTIME_CMD stop ${CONTAINER_NAME} $RUNTIME_CMD rm ${CONTAINER_NAME} print_info "Production container stopped and removed." @@ -226,4 +217,4 @@ case "${1:-help}" in show_usage exit 1 ;; -esac \ No newline at end of file +esac diff --git a/supervisord.conf b/supervisord.conf deleted file mode 100644 index df41bd1..0000000 --- a/supervisord.conf +++ /dev/null @@ -1,38 +0,0 @@ -[supervisord] -nodaemon=true -logfile=/var/log/supervisor/supervisord.log -pidfile=/tmp/supervisord.pid - -[program:chromadb] -command=python -m chromadb.cli.cli run --path /app/chroma --host 0.0.0.0 --port 8001 -directory=/app -autostart=true -autorestart=true -priority=1 -stdout_logfile=/app/data/logs/chromadb.stdout.log -stderr_logfile=/app/data/logs/chromadb.stderr.log -environment=IS_PERSISTENT="TRUE",ANONYMIZED_TELEMETRY="FALSE",ALLOW_RESET="FALSE" - -[program:documentmanager] -command=python -m uvicorn app.main:app --host 0.0.0.0 --port 8000 --proxy-headers --forwarded-allow-ips=%(ENV_TRUSTED_PROXY_IPS)s -directory=/app -autostart=true -autorestart=true -priority=2 -startsecs=10 -stdout_logfile=/app/data/logs/documentmanager.stdout.log -stderr_logfile=/app/data/logs/documentmanager.stderr.log -environment=CHROMADB_HOST="localhost",CHROMADB_PORT="8001" - -[group:services] -programs=chromadb,documentmanager - -[unix_http_server] -file=/tmp/supervisor.sock -chmod=0700 - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisorctl] -serverurl=unix:///tmp/supervisor.sock diff --git a/tests/test_backup_security.py b/tests/test_backup_security.py new file mode 100644 index 0000000..e4f078a --- /dev/null +++ b/tests/test_backup_security.py @@ -0,0 +1,73 @@ +import io +import tarfile + +import pytest + +from app.config import Settings +from app.utils.backup import BackupError, _extract_tar_safely, backup_files + + +def add_bytes(tar: tarfile.TarFile, name: str, payload: bytes) -> None: + member = tarfile.TarInfo(name) + member.size = len(payload) + tar.addfile(member, io.BytesIO(payload)) + + +def test_extract_tar_safely_extracts_regular_files(tmp_path): + archive = tmp_path / "safe.tar" + destination = tmp_path / "destination" + destination.mkdir() + + with tarfile.open(archive, "w") as tar: + add_bytes(tar, "backup/metadata.json", b"{}") + + with tarfile.open(archive) as tar: + _extract_tar_safely(tar, destination) + + assert (destination / "backup" / "metadata.json").read_bytes() == b"{}" + + +def test_extract_tar_safely_rejects_path_traversal(tmp_path): + archive = tmp_path / "traversal.tar" + destination = tmp_path / "destination" + destination.mkdir() + + with tarfile.open(archive, "w") as tar: + add_bytes(tar, "../outside.txt", b"not allowed") + + with tarfile.open(archive) as tar: + with pytest.raises(BackupError, match="outside dest"): + _extract_tar_safely(tar, destination) + + assert not (tmp_path / "outside.txt").exists() + + +def test_extract_tar_safely_rejects_links(tmp_path): + archive = tmp_path / "link.tar" + destination = tmp_path / "destination" + destination.mkdir() + + with tarfile.open(archive, "w") as tar: + link = tarfile.TarInfo("backup/link") + link.type = tarfile.SYMTYPE + link.linkname = "metadata.json" + tar.addfile(link) + + with tarfile.open(archive) as tar: + with pytest.raises(BackupError, match="tar link"): + _extract_tar_safely(tar, destination) + + +def test_backup_files_uses_typed_settings_paths(tmp_path): + storage = tmp_path / "storage" + staging = tmp_path / "staging" + storage.mkdir() + staging.mkdir() + (storage / "document.txt").write_text("stored", encoding="utf-8") + (staging / "pending.txt").write_text("pending", encoding="utf-8") + settings = Settings(storage_folder=str(storage), staging_folder=str(staging)) + + result = backup_files(tmp_path / "backup", settings) + + assert (result / "storage" / "document.txt").read_text(encoding="utf-8") == "stored" + assert (result / "staging" / "pending.txt").read_text(encoding="utf-8") == "pending" diff --git a/tests/test_runtime_configuration.py b/tests/test_runtime_configuration.py index 66519c6..af08598 100644 --- a/tests/test_runtime_configuration.py +++ b/tests/test_runtime_configuration.py @@ -1,9 +1,13 @@ +from pathlib import Path from types import SimpleNamespace from app.config import Settings from app.middleware.rate_limit_middleware import RateLimitMiddleware +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] + + def test_settings_read_documented_environment_variables(monkeypatch): monkeypatch.setenv("DATABASE_URL", "sqlite:///./data/test.db") monkeypatch.setenv("OPENAI_API_KEY", "test-key") @@ -38,3 +42,54 @@ def test_forwarded_header_is_used_for_trusted_proxy(): ) assert middleware.get_client_ip(request) == "198.51.100.7" + + +def test_env_example_only_documents_supported_settings(): + env_keys = { + line.split("=", 1)[0] + for line in (REPOSITORY_ROOT / ".env.example").read_text(encoding="utf-8").splitlines() + if line and not line.startswith("#") + } + supported_keys = {name.upper() for name in Settings.model_fields} + + assert env_keys <= supported_keys + assert { + "SECRET_KEY", + "DATABASE_URL", + "AI_PROVIDER", + "OPENAI_API_KEY", + "AZURE_OPENAI_API_KEY", + "TRUSTED_PROXY_IPS", + } <= env_keys + + +def test_setup_uses_env_file_without_evaluating_its_contents(): + setup_script = (REPOSITORY_ROOT / "setup.sh").read_text(encoding="utf-8") + + assert "--env-file .env" in setup_script + assert "export $(" not in setup_script + + +def test_container_runtime_does_not_start_a_duplicate_chroma_server(): + production_dockerfile = (REPOSITORY_ROOT / "Dockerfile").read_text(encoding="utf-8") + development_dockerfile = (REPOSITORY_ROOT / "Dockerfile.dev").read_text(encoding="utf-8") + entrypoint = (REPOSITORY_ROOT / "docker-entrypoint.sh").read_text(encoding="utf-8") + + assert "supervisor" not in production_dockerfile + assert "supervisor" not in development_dockerfile + assert "chromadb.cli" not in entrypoint + assert "python -m uvicorn" in entrypoint + + +def test_runtime_dependencies_do_not_pull_unused_local_embedding_stack(): + requirements = (REPOSITORY_ROOT / "requirements.txt").read_text(encoding="utf-8") + + assert "sentence-transformers" not in requirements + + +def test_docker_context_excludes_credentials_and_runtime_data(): + dockerignore = (REPOSITORY_ROOT / ".dockerignore").read_text(encoding="utf-8").splitlines() + + assert ".env" in dockerignore + assert "data" in dockerignore + assert "backups" in dockerignore