diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e7cf968..c918a19 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,86 +1,74 @@ -name: Release - -on: - push: - tags: - - "[0-9]+.[0-9]+.[0-9]+*" # Trigger on version tags like 1.0.0, 2.1.0, etc. - -jobs: - release: - runs-on: [blacksmith-8vcpu-ubuntu-2204, linux] - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Clear space to remove unused folders - run: | - rm -rf /usr/share/dotnet - rm -rf /opt/ghc - rm -rf "/usr/local/share/boost" - rm -rf "$AGENT_TOOLSDIRECTORY" - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: blacksmith docker layer cache - uses: useblacksmith/build-push-action@v1 - with: - setup-only: true - - - name: Set environment variables - run: | - echo "DOCKERHUB_REPO=${{ vars.DOCKERHUB_REPO || 'runpod' }}" >> $GITHUB_ENV - echo "DOCKERHUB_IMG=${{ vars.DOCKERHUB_IMG || 'worker-v1-vllm' }}" >> $GITHUB_ENV - echo "HUGGINGFACE_ACCESS_TOKEN=${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}" >> $GITHUB_ENV - - # For tags, use the tag name as version (remove refs/tags/ prefix) - VERSION=${GITHUB_REF#refs/tags/} - echo "RELEASE_VERSION=${VERSION}" >> $GITHUB_ENV - - - name: Build and push the images to Docker Hub - uses: docker/bake-action@v2 - with: - push: true - set: | - *.args.DOCKERHUB_REPO=${{ env.DOCKERHUB_REPO }} - *.args.DOCKERHUB_IMG=${{ env.DOCKERHUB_IMG }} - *.args.RELEASE_VERSION=${{ env.RELEASE_VERSION }} - *.args.HUGGINGFACE_ACCESS_TOKEN=${{ env.HUGGINGFACE_ACCESS_TOKEN }} - - - name: Build and push latest tag - uses: docker/bake-action@v2 - with: - push: true - set: | - *.args.DOCKERHUB_REPO=${{ env.DOCKERHUB_REPO }} - *.args.DOCKERHUB_IMG=${{ env.DOCKERHUB_IMG }} - *.args.RELEASE_VERSION=latest - *.args.HUGGINGFACE_ACCESS_TOKEN=${{ env.HUGGINGFACE_ACCESS_TOKEN }} - - - name: Create GitHub Release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref_name }} - release_name: Release ${{ github.ref_name }} - body: | - Release ${{ github.ref_name }} - - Docker Images: - - `${{ env.DOCKERHUB_REPO }}/${{ env.DOCKERHUB_IMG }}:${{ env.RELEASE_VERSION }}` - - `${{ env.DOCKERHUB_REPO }}/${{ env.DOCKERHUB_IMG }}:latest` - - ## Changes - See [commit history](https://github.com/${{ github.repository }}/commits/${{ github.ref_name }}) for detailed changes. - draft: false - prerelease: false +name: Release + +on: + push: + tags: + - "[0-9]+.[0-9]+.[0-9]+*" # Trigger on version tags like 1.0.0, 2.1.0, etc. + +jobs: + release: + runs-on: [blacksmith-8vcpu-ubuntu-2204, linux] + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Clear space to remove unused folders + run: | + rm -rf /usr/share/dotnet + rm -rf /opt/ghc + rm -rf "/usr/local/share/boost" + rm -rf "$AGENT_TOOLSDIRECTORY" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: blacksmith docker layer cache + uses: useblacksmith/build-push-action@v1 + with: + setup-only: true + + - name: Set environment variables + run: | + echo "DOCKERHUB_REPO=${{ vars.DOCKERHUB_REPO || 'runpod' }}" >> $GITHUB_ENV + echo "DOCKERHUB_IMG=${{ vars.DOCKERHUB_IMG || 'worker-v1-vllm' }}" >> $GITHUB_ENV + echo "HUGGINGFACE_ACCESS_TOKEN=${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}" >> $GITHUB_ENV + + # For tags, use the tag name as version (remove refs/tags/ prefix) + VERSION=${GITHUB_REF#refs/tags/} + echo "RELEASE_VERSION=${VERSION}" >> $GITHUB_ENV + + - name: Build and push the images to Docker Hub + uses: docker/bake-action@v2 + with: + push: true + set: | + *.args.DOCKERHUB_REPO=${{ env.DOCKERHUB_REPO }} + *.args.DOCKERHUB_IMG=${{ env.DOCKERHUB_IMG }} + *.args.RELEASE_VERSION=${{ env.RELEASE_VERSION }} + *.args.HUGGINGFACE_ACCESS_TOKEN=${{ env.HUGGINGFACE_ACCESS_TOKEN }} + + - name: Create GitHub Release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref_name }} + release_name: Release ${{ github.ref_name }} + body: | + Release ${{ github.ref_name }} + + Docker Image: `${{ env.DOCKERHUB_REPO }}/${{ env.DOCKERHUB_IMG }}:${{ env.RELEASE_VERSION }}` + + ## Changes + See [commit history](https://github.com/${{ github.repository }}/commits/${{ github.ref_name }}) for detailed changes. + draft: false + prerelease: false diff --git a/README.md b/README.md index 5f676f2..82b73e9 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https: ### 2. Worker vLLM with latest vLLM version now available -The latest Worker vLLM is now available. Use the image tag `runpod/worker-v1-vllm:latest` for the most recent release, or check [Docker Hub](https://hub.docker.com/r/runpod/worker-v1-vllm/tags) for specific version tags. +The latest Worker vLLM is now available. See [GitHub Releases](https://github.com/runpod-workers/worker-vllm/releases) for available versions. ### 3. OpenAI-Compatible [Embedding Worker](https://github.com/runpod-workers/worker-infinity-embedding) Released @@ -80,9 +80,9 @@ We now offer a pre-built Docker Image for the vLLM Worker that you can configure Below is a summary of the available RunPod Worker images, categorized by image stability and CUDA version compatibility. -| CUDA Version | Stable Image Tag | Development Image Tag | Note | -| ------------ | ------------------------------ | ----------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 12.1.0 | `runpod/worker-v1-vllm:latest` | `runpod/worker-v1-vllm:dev-` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. Use `latest` for stable releases or specific version tags like `2.7.0` | +| CUDA Version | Stable Image Tag | Development Image Tag | Note | +| ------------ | --------------------------------- | ----------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 12.1.0 | `runpod/worker-v1-vllm:` | `runpod/worker-v1-vllm:dev-` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. See [GitHub Releases](https://github.com/runpod-workers/worker-vllm/releases) for available versions | --- @@ -652,7 +652,7 @@ The JSON consists of two main parts, schema and versions. { "versions": { "0.5.4": { - "imageName": "runpod/worker-v1-vllm:1.2.0stable-cuda12.1.0", + "imageName": "runpod/worker-v1-vllm:", "minimumCudaVersion": "12.1", "categories": [ { diff --git a/docs/conventions.md b/docs/conventions.md index ee7664b..eec112d 100644 --- a/docs/conventions.md +++ b/docs/conventions.md @@ -1,343 +1,342 @@ -# Worker vLLM - Development Conventions & Architecture Guide - -## Project Overview - -**worker-vllm** is a RunPod serverless worker that provides OpenAI-compatible endpoints for Large Language Model (LLM) inference, powered by the vLLM engine. It enables blazing-fast LLM deployment on RunPod's serverless infrastructure with minimal configuration. - -### Core Purpose - -- **Primary Function**: Deploy any Hugging Face LLM as an OpenAI-compatible API endpoint -- **Platform**: RunPod Serverless infrastructure -- **Engine**: vLLM (high-performance LLM inference engine) -- **Compatibility**: Drop-in replacement for OpenAI API (Chat Completions, Models) - -## High-Level Architecture - -### 1. **Entry Point & Request Flow** - -``` -RunPod Request → handler.py → JobInput → Engine Selection → vLLM Generation → Streaming Response -``` - -**Key Components:** - -- `src/handler.py`: Main entry point using RunPod serverless framework -- `src/utils.py`: Request parsing and utility classes (`JobInput`, `BatchSize`) -- Two engine modes: OpenAI-compatible vs. standard vLLM - -### 2. **Engine Architecture** - -#### Core Classes: - -- **`vLLMEngine`**: Base engine handling vLLM initialization and generation -- **`OpenAIvLLMEngine`**: Wrapper providing OpenAI API compatibility -- **Engine Selection**: Automatic routing based on `job_input.openai_route` - -#### Key Design Patterns: - -- **Dual API Support**: Same codebase serves both OpenAI-compatible and native vLLM APIs -- **Streaming by Default**: Token-level streaming with configurable batching -- **Dynamic Batching**: Adaptive batch sizes that grow from min → max for efficiency - -### 3. **Configuration System** - -#### Environment-Based Configuration: - -- **Single Source of Truth**: All configuration via environment variables -- **Hierarchical Loading**: `DEFAULT_ARGS` → `os.environ` → `local_model_args.json` (for baked models) -- **vLLM Argument Mapping**: Automatic translation of env vars to vLLM `AsyncEngineArgs` - -#### Key Configuration Files: - -- `src/engine_args.py`: Centralized configuration management -- `src/constants.py`: Default values for core settings -- `worker-config.json`: UI form generation for RunPod console - -## Core Development Concepts - -### 1. **Deployment Models** - -#### Option 1: Pre-built Images (Recommended) - -- **Image**: `runpod/worker-v1-vllm:latest` (or specific version tags like `2.7.0`) -- **Configuration**: Entirely via environment variables -- **Model Loading**: Downloads model at runtime from Hugging Face -- **Use Case**: Quick deployment, model experimentation - -#### Option 2: Baked Model Images - -- **Build Process**: Model downloaded during Docker build -- **Storage**: Model embedded in container image -- **Configuration**: Stored in `/local_model_args.json` -- **Use Case**: Production deployments, faster cold starts - -### 2. **Request Processing Patterns** - -#### Input Handling: - -```python -class JobInput: - - llm_input: str | List[Dict] (prompt or messages) - - sampling_params: SamplingParams (generation settings) - - stream: bool (streaming vs batch response) - - openai_route: bool (API compatibility mode) - - batch_size configs: Dynamic batching parameters -``` - -#### Response Streaming: - -- **Batched Streaming**: Tokens grouped into configurable batch sizes -- **Dynamic Growth**: `min_batch_size * growth_factor^n` up to `max_batch_size` -- **Usage Tracking**: Input/output token counting for billing - -### 3. **Model & Tokenizer Management** - -#### Tokenizer Handling: - -- **Wrapper Pattern**: `TokenizerWrapper` for consistent chat template application -- **Special Cases**: Mistral models use vLLM's native tokenizer -- **Chat Templates**: Automatic application for message-based inputs - -#### Model Loading: - -- **Multi-GPU Support**: Automatic tensor parallelism detection -- **Quantization**: Support for AWQ, GPTQ, BitsAndBytes -- **Caching**: Hugging Face cache management - -## Development Patterns & Best Practices - -### 1. **Code Organization** - -#### File Structure: - -``` -src/ -├── handler.py # RunPod entry point -├── engine.py # Core vLLM engines -├── engine_args.py # Configuration management -├── utils.py # Request parsing & utilities -├── tokenizer.py # Tokenizer wrapper -├── constants.py # Default constants -└── download_model.py # Model downloading logic -``` - -#### Separation of Concerns: - -- **Engine Logic**: Isolated in `engine.py` classes -- **Configuration**: Centralized in `engine_args.py` -- **Request Handling**: Abstracted via `JobInput` class -- **Platform Integration**: Contained in `handler.py` - -### 2. **Error Handling & Logging** - -#### Logging Strategy: - -- **Structured Logging**: Consistent format across components -- **Performance Tracking**: Timer decorators for critical operations -- **Error Context**: Detailed error messages with configuration context - -#### Error Responses: - -- **OpenAI Compatibility**: Standard OpenAI error format -- **Graceful Degradation**: Fallback behaviors for edge cases - -### 3. **Environment Variable Conventions** - -#### Naming Patterns: - -- **vLLM Settings**: Match vLLM parameter names (uppercase) -- **RunPod Settings**: `MAX_CONCURRENCY`, `DEFAULT_BATCH_SIZE` -- **OpenAI Settings**: `OPENAI_` prefix for compatibility settings -- **Feature Flags**: `ENABLE_*`, `DISABLE_*` pattern - -#### Type Conventions: - -- **Booleans**: String 'true'/'false' or int 0/1 -- **Lists**: Comma-separated strings -- **Objects**: JSON strings for complex configurations - -### 4. **Docker & Deployment** - -#### Multi-Stage Builds: - -- **Base**: CUDA runtime environment -- **Dependencies**: Python packages and vLLM -- **Model Download**: Optional model baking stage -- **Runtime**: Final application layer - -#### Build Arguments: - -- **MODEL_NAME**: Primary model identifier -- **BASE_PATH**: Storage location strategy -- **QUANTIZATION**: Optimization settings -- **WORKER_CUDA_VERSION**: CUDA compatibility - -#### CI/CD Strategy: - -- **Development Builds**: All non-main branches → `runpod/worker-v1-vllm:dev-` -- **Release Builds**: Git tags (numeric) only → `runpod/worker-v1-vllm:` + `latest` -- **Dependency Updates**: Automated runpod package version monitoring - -#### Docker Bake Configuration: - -- **File**: `docker-bake.hcl` (flexible variable-based configuration) -- **Variables**: `DOCKERHUB_REPO`, `DOCKERHUB_IMG`, `RELEASE_VERSION`, `HUGGINGFACE_ACCESS_TOKEN` -- **Platform**: `linux/amd64` (GPU-optimized) - -## Release & Versioning Strategy - -### 1. **Version Tagging** - -- **Development**: `dev-` (e.g., `dev-feature-new-api`) -- **Latest Release**: `latest` (always points to most recent stable release) -- **Specific Versions**: `2.7.0`, `2.8.0` (semantic versioning without "v" prefix) -- **Date Snapshots**: `20241215` (YYYYMMDD format for main branch builds) - -### 2. **Release Workflow** - -1. **Feature Development**: Work on feature branches → triggers dev builds -2. **Main Branch Staging**: Merge features to main → stable codebase (no builds) -3. **Version Release**: Create git tag from main branch (e.g., `2.8.0`) → triggers versioned release + `latest` + GitHub release -4. **Docker Hub**: All builds pushed with appropriate tags - -### 3. **Branch Strategy** - -- **Feature Branches**: `feature/*`, `fix/*`, `feat/*` etc. → Dev builds -- **Main Branch**: Stable codebase ready for release (no automatic builds) -- **Git Tags**: Must be created from main branch for formal version releases - -### 4. **Deployment Recommendations** - -- **Production**: Use specific version tags (e.g., `2.7.0`) for stability -- **Staging**: Use `latest` for recent stable features -- **Development**: Use `dev-` for testing specific features -- **Release Process**: Always tag from main branch: `git checkout main && git tag 2.8.0 && git push origin 2.8.0` - -## Performance & Scaling Considerations - -### 1. **Memory Management** - -- **GPU Utilization**: Default 95% GPU memory utilization -- **KV Cache**: Configurable cache types (auto, fp8) -- **Swap Space**: CPU offloading for large contexts - -### 2. **Concurrency Patterns** - -- **Max Concurrency**: 300 concurrent requests by default -- **vLLM Queuing**: Internal request batching and scheduling -- **RunPod Integration**: Concurrency modifier for auto-scaling - -### 3. **Optimization Features** - -- **Prefix Caching**: Automatic caching of common prefixes -- **Speculative Decoding**: Draft model acceleration -- **Chunked Prefill**: Memory-efficient long context handling - -## Testing & Development - -### 1. **Local Development** - -- **Environment**: Virtual environment with GPU support -- **Configuration**: `.env` files for local testing -- **Model Testing**: Small models for development (facebook/opt-125m) - -### 2. **Docker Development** - -- **Build Strategy**: `docker-bake.hcl` for consistent builds -- **Testing Images**: Separate dev/stable image tags -- **Layer Caching**: Optimized for rapid iteration - -### 3. **Configuration Validation** - -- **Argument Matching**: Automatic validation against vLLM parameters -- **Environment Validation**: Type checking and default value handling -- **Runtime Validation**: Model compatibility checks - -## API Conventions - -### 1. **OpenAI Compatibility** - -- **Endpoint Mapping**: `/openai/v1/chat/completions`, `/openai/v1/models` -- **Request Format**: Exact OpenAI request/response schemas -- **Authentication**: RunPod API key in Authorization header -- **Model Names**: Hugging Face repo names or custom overrides - -### 2. **Native vLLM API** - -- **Input Format**: `prompt` or `messages` with `sampling_params` -- **Streaming**: Token-level streaming with configurable batching -- **Extensibility**: Support for vLLM-specific features - -## Common Patterns & Utilities - -### 1. **Configuration Loading** - -```python -# Standard pattern for new configuration options -def get_engine_args(): - args = DEFAULT_ARGS - args.update(os.environ) # Environment override - args.update(get_local_args()) # Baked model override - return match_vllm_args(args) # Validate against vLLM -``` - -### 2. **Error Handling** - -```python -# Standard error response pattern -def create_error_response(message: str, err_type: str = "BadRequestError"): - return ErrorResponse(message=message, type=err_type) -``` - -### 3. **Async Generation** - -```python -# Standard streaming pattern -async def generate(self, job_input: JobInput): - async for batch in self._generate_vllm(...): - yield batch # Batch-level yielding for efficiency -``` - -## Extension Points - -### 1. **New Model Architectures** - -- **Engine Args**: Add new parameters in `engine_args.py` -- **Compatibility**: Update vLLM argument mapping -- **Validation**: Add architecture-specific validation - -### 2. **New API Features** - -- **Engine Extension**: Extend `vLLMEngine` or `OpenAIvLLMEngine` -- **Input Parsing**: Extend `JobInput` class -- **Response Format**: Add new response generators - -### 3. **Performance Optimizations** - -- **Batching Strategy**: Modify `BatchSize` class -- **Memory Management**: Add new caching strategies -- **Hardware Optimization**: GPU-specific optimizations - -## Security & Best Practices - -### 1. **Secret Management** - -- **Build Secrets**: Docker secrets for HF tokens -- **Runtime Secrets**: Environment variable injection -- **Token Handling**: Secure authentication patterns - -### 2. **Resource Limits** - -- **Memory Bounds**: Configurable GPU memory limits -- **Request Limits**: Concurrency and timeout controls -- **Model Safety**: Trust remote code flags - -### 3. **Logging Security** - -- **Sanitization**: No secrets in logs -- **Request Logging**: Configurable request/response logging -- **Performance Monitoring**: Safe metrics collection - ---- - -This guide should be consulted whenever working on the worker-vllm codebase to ensure consistency with established patterns and architectural decisions. +# Worker vLLM - Development Conventions & Architecture Guide + +## Project Overview + +**worker-vllm** is a RunPod serverless worker that provides OpenAI-compatible endpoints for Large Language Model (LLM) inference, powered by the vLLM engine. It enables blazing-fast LLM deployment on RunPod's serverless infrastructure with minimal configuration. + +### Core Purpose + +- **Primary Function**: Deploy any Hugging Face LLM as an OpenAI-compatible API endpoint +- **Platform**: RunPod Serverless infrastructure +- **Engine**: vLLM (high-performance LLM inference engine) +- **Compatibility**: Drop-in replacement for OpenAI API (Chat Completions, Models) + +## High-Level Architecture + +### 1. **Entry Point & Request Flow** + +``` +RunPod Request → handler.py → JobInput → Engine Selection → vLLM Generation → Streaming Response +``` + +**Key Components:** + +- `src/handler.py`: Main entry point using RunPod serverless framework +- `src/utils.py`: Request parsing and utility classes (`JobInput`, `BatchSize`) +- Two engine modes: OpenAI-compatible vs. standard vLLM + +### 2. **Engine Architecture** + +#### Core Classes: + +- **`vLLMEngine`**: Base engine handling vLLM initialization and generation +- **`OpenAIvLLMEngine`**: Wrapper providing OpenAI API compatibility +- **Engine Selection**: Automatic routing based on `job_input.openai_route` + +#### Key Design Patterns: + +- **Dual API Support**: Same codebase serves both OpenAI-compatible and native vLLM APIs +- **Streaming by Default**: Token-level streaming with configurable batching +- **Dynamic Batching**: Adaptive batch sizes that grow from min → max for efficiency + +### 3. **Configuration System** + +#### Environment-Based Configuration: + +- **Single Source of Truth**: All configuration via environment variables +- **Hierarchical Loading**: `DEFAULT_ARGS` → `os.environ` → `local_model_args.json` (for baked models) +- **vLLM Argument Mapping**: Automatic translation of env vars to vLLM `AsyncEngineArgs` + +#### Key Configuration Files: + +- `src/engine_args.py`: Centralized configuration management +- `src/constants.py`: Default values for core settings +- `worker-config.json`: UI form generation for RunPod console + +## Core Development Concepts + +### 1. **Deployment Models** + +#### Option 1: Pre-built Images (Recommended) + +- **Image**: `runpod/worker-v1-vllm:` (see [GitHub Releases](https://github.com/runpod-workers/worker-vllm/releases)) +- **Configuration**: Entirely via environment variables +- **Model Loading**: Downloads model at runtime from Hugging Face +- **Use Case**: Quick deployment, model experimentation + +#### Option 2: Baked Model Images + +- **Build Process**: Model downloaded during Docker build +- **Storage**: Model embedded in container image +- **Configuration**: Stored in `/local_model_args.json` +- **Use Case**: Production deployments, faster cold starts + +### 2. **Request Processing Patterns** + +#### Input Handling: + +```python +class JobInput: + - llm_input: str | List[Dict] (prompt or messages) + - sampling_params: SamplingParams (generation settings) + - stream: bool (streaming vs batch response) + - openai_route: bool (API compatibility mode) + - batch_size configs: Dynamic batching parameters +``` + +#### Response Streaming: + +- **Batched Streaming**: Tokens grouped into configurable batch sizes +- **Dynamic Growth**: `min_batch_size * growth_factor^n` up to `max_batch_size` +- **Usage Tracking**: Input/output token counting for billing + +### 3. **Model & Tokenizer Management** + +#### Tokenizer Handling: + +- **Wrapper Pattern**: `TokenizerWrapper` for consistent chat template application +- **Special Cases**: Mistral models use vLLM's native tokenizer +- **Chat Templates**: Automatic application for message-based inputs + +#### Model Loading: + +- **Multi-GPU Support**: Automatic tensor parallelism detection +- **Quantization**: Support for AWQ, GPTQ, BitsAndBytes +- **Caching**: Hugging Face cache management + +## Development Patterns & Best Practices + +### 1. **Code Organization** + +#### File Structure: + +``` +src/ +├── handler.py # RunPod entry point +├── engine.py # Core vLLM engines +├── engine_args.py # Configuration management +├── utils.py # Request parsing & utilities +├── tokenizer.py # Tokenizer wrapper +├── constants.py # Default constants +└── download_model.py # Model downloading logic +``` + +#### Separation of Concerns: + +- **Engine Logic**: Isolated in `engine.py` classes +- **Configuration**: Centralized in `engine_args.py` +- **Request Handling**: Abstracted via `JobInput` class +- **Platform Integration**: Contained in `handler.py` + +### 2. **Error Handling & Logging** + +#### Logging Strategy: + +- **Structured Logging**: Consistent format across components +- **Performance Tracking**: Timer decorators for critical operations +- **Error Context**: Detailed error messages with configuration context + +#### Error Responses: + +- **OpenAI Compatibility**: Standard OpenAI error format +- **Graceful Degradation**: Fallback behaviors for edge cases + +### 3. **Environment Variable Conventions** + +#### Naming Patterns: + +- **vLLM Settings**: Match vLLM parameter names (uppercase) +- **RunPod Settings**: `MAX_CONCURRENCY`, `DEFAULT_BATCH_SIZE` +- **OpenAI Settings**: `OPENAI_` prefix for compatibility settings +- **Feature Flags**: `ENABLE_*`, `DISABLE_*` pattern + +#### Type Conventions: + +- **Booleans**: String 'true'/'false' or int 0/1 +- **Lists**: Comma-separated strings +- **Objects**: JSON strings for complex configurations + +### 4. **Docker & Deployment** + +#### Multi-Stage Builds: + +- **Base**: CUDA runtime environment +- **Dependencies**: Python packages and vLLM +- **Model Download**: Optional model baking stage +- **Runtime**: Final application layer + +#### Build Arguments: + +- **MODEL_NAME**: Primary model identifier +- **BASE_PATH**: Storage location strategy +- **QUANTIZATION**: Optimization settings +- **WORKER_CUDA_VERSION**: CUDA compatibility + +#### CI/CD Strategy: + +- **Development Builds**: All non-main branches → `runpod/worker-v1-vllm:dev-` +- **Release Builds**: Git tags (numeric) only → `runpod/worker-v1-vllm:` +- **Dependency Updates**: Automated runpod package version monitoring + +#### Docker Bake Configuration: + +- **File**: `docker-bake.hcl` (flexible variable-based configuration) +- **Variables**: `DOCKERHUB_REPO`, `DOCKERHUB_IMG`, `RELEASE_VERSION`, `HUGGINGFACE_ACCESS_TOKEN` +- **Platform**: `linux/amd64` (GPU-optimized) + +## Release & Versioning Strategy + +### 1. **Version Tagging** + +- **Development**: `dev-` (e.g., `dev-feature-new-api`) +- **Specific Versions**: `2.7.0`, `2.8.0` (semantic versioning without "v" prefix) +- **Version Discovery**: Check [GitHub Releases](https://github.com/runpod-workers/worker-vllm/releases) for available versions + +### 2. **Release Workflow** + +1. **Feature Development**: Work on feature branches → triggers dev builds +2. **Main Branch Staging**: Merge features to main → stable codebase (no builds) +3. **Version Release**: Create git tag from main branch (e.g., `2.8.0`) → triggers versioned release + GitHub release +4. **Docker Hub**: Versioned image pushed with tag + +### 3. **Branch Strategy** + +- **Feature Branches**: `feature/*`, `fix/*`, `feat/*` etc. → Dev builds +- **Main Branch**: Stable codebase ready for release (no automatic builds) +- **Git Tags**: Must be created from main branch for formal version releases + +### 4. **Deployment Recommendations** + +- **Production**: Use specific version tags (e.g., `2.7.0`) for stability +- **Development**: Use `dev-` for testing specific features +- **Version Selection**: Check [GitHub Releases](https://github.com/runpod-workers/worker-vllm/releases) for available versions +- **Release Process**: Always tag from main branch: `git checkout main && git tag 2.8.0 && git push origin 2.8.0` + +## Performance & Scaling Considerations + +### 1. **Memory Management** + +- **GPU Utilization**: Default 95% GPU memory utilization +- **KV Cache**: Configurable cache types (auto, fp8) +- **Swap Space**: CPU offloading for large contexts + +### 2. **Concurrency Patterns** + +- **Max Concurrency**: 300 concurrent requests by default +- **vLLM Queuing**: Internal request batching and scheduling +- **RunPod Integration**: Concurrency modifier for auto-scaling + +### 3. **Optimization Features** + +- **Prefix Caching**: Automatic caching of common prefixes +- **Speculative Decoding**: Draft model acceleration +- **Chunked Prefill**: Memory-efficient long context handling + +## Testing & Development + +### 1. **Local Development** + +- **Environment**: Virtual environment with GPU support +- **Configuration**: `.env` files for local testing +- **Model Testing**: Small models for development (facebook/opt-125m) + +### 2. **Docker Development** + +- **Build Strategy**: `docker-bake.hcl` for consistent builds +- **Testing Images**: Separate dev/stable image tags +- **Layer Caching**: Optimized for rapid iteration + +### 3. **Configuration Validation** + +- **Argument Matching**: Automatic validation against vLLM parameters +- **Environment Validation**: Type checking and default value handling +- **Runtime Validation**: Model compatibility checks + +## API Conventions + +### 1. **OpenAI Compatibility** + +- **Endpoint Mapping**: `/openai/v1/chat/completions`, `/openai/v1/models` +- **Request Format**: Exact OpenAI request/response schemas +- **Authentication**: RunPod API key in Authorization header +- **Model Names**: Hugging Face repo names or custom overrides + +### 2. **Native vLLM API** + +- **Input Format**: `prompt` or `messages` with `sampling_params` +- **Streaming**: Token-level streaming with configurable batching +- **Extensibility**: Support for vLLM-specific features + +## Common Patterns & Utilities + +### 1. **Configuration Loading** + +```python +# Standard pattern for new configuration options +def get_engine_args(): + args = DEFAULT_ARGS + args.update(os.environ) # Environment override + args.update(get_local_args()) # Baked model override + return match_vllm_args(args) # Validate against vLLM +``` + +### 2. **Error Handling** + +```python +# Standard error response pattern +def create_error_response(message: str, err_type: str = "BadRequestError"): + return ErrorResponse(message=message, type=err_type) +``` + +### 3. **Async Generation** + +```python +# Standard streaming pattern +async def generate(self, job_input: JobInput): + async for batch in self._generate_vllm(...): + yield batch # Batch-level yielding for efficiency +``` + +## Extension Points + +### 1. **New Model Architectures** + +- **Engine Args**: Add new parameters in `engine_args.py` +- **Compatibility**: Update vLLM argument mapping +- **Validation**: Add architecture-specific validation + +### 2. **New API Features** + +- **Engine Extension**: Extend `vLLMEngine` or `OpenAIvLLMEngine` +- **Input Parsing**: Extend `JobInput` class +- **Response Format**: Add new response generators + +### 3. **Performance Optimizations** + +- **Batching Strategy**: Modify `BatchSize` class +- **Memory Management**: Add new caching strategies +- **Hardware Optimization**: GPU-specific optimizations + +## Security & Best Practices + +### 1. **Secret Management** + +- **Build Secrets**: Docker secrets for HF tokens +- **Runtime Secrets**: Environment variable injection +- **Token Handling**: Secure authentication patterns + +### 2. **Resource Limits** + +- **Memory Bounds**: Configurable GPU memory limits +- **Request Limits**: Concurrency and timeout controls +- **Model Safety**: Trust remote code flags + +### 3. **Logging Security** + +- **Sanitization**: No secrets in logs +- **Request Logging**: Configurable request/response logging +- **Performance Monitoring**: Safe metrics collection + +--- + +This guide should be consulted whenever working on the worker-vllm codebase to ensure consistency with established patterns and architectural decisions.