266 lines
7.2 KiB
Python
266 lines
7.2 KiB
Python
"""
|
|
Configuration management for RunPod workflow using pydantic-settings.
|
|
|
|
Loads configuration from .env.runpod file with validation.
|
|
"""
|
|
|
|
from pathlib import Path
|
|
from typing import Literal
|
|
|
|
from pydantic import Field, field_validator
|
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
|
|
from .errors import ConfigurationError
|
|
|
|
|
|
class RunPodConfig(BaseSettings):
|
|
"""
|
|
RunPod configuration with validation.
|
|
|
|
All settings can be overridden via environment variables.
|
|
Default configuration file: .env.runpod in project root.
|
|
"""
|
|
|
|
# RunPod API
|
|
runpod_api_key: str = Field(
|
|
...,
|
|
description="RunPod API key for pod management",
|
|
min_length=32,
|
|
)
|
|
|
|
# RunPod S3 Credentials
|
|
runpod_s3_access_key: str = Field(
|
|
...,
|
|
description="RunPod S3 access key for network volume",
|
|
min_length=16,
|
|
)
|
|
runpod_s3_secret: str = Field(
|
|
...,
|
|
description="RunPod S3 secret key",
|
|
min_length=32,
|
|
)
|
|
runpod_s3_endpoint: str = Field(
|
|
default="https://s3api-eur-is-1.runpod.io",
|
|
description="RunPod S3 endpoint URL",
|
|
)
|
|
runpod_s3_region: str = Field(
|
|
default="eur-is-1",
|
|
description="RunPod S3 region",
|
|
)
|
|
|
|
# Network Volume
|
|
runpod_volume_id: str = Field(
|
|
...,
|
|
description="RunPod network volume ID (bucket name)",
|
|
min_length=10,
|
|
)
|
|
runpod_volume_mount: str = Field(
|
|
default="/runpod-volume",
|
|
description="Volume mount path in containers",
|
|
)
|
|
|
|
# Pod Configuration
|
|
runpod_gpu_type: str = Field(
|
|
default="NVIDIA RTX 4090",
|
|
description="Preferred GPU type for deployment",
|
|
)
|
|
runpod_image: str = Field(
|
|
default="jgrusewski/foxhunt:latest",
|
|
description="Docker image to deploy",
|
|
)
|
|
runpod_container_registry_auth_id: str | None = Field(
|
|
default=None,
|
|
description="Container registry auth ID for private images",
|
|
)
|
|
|
|
# Deployment Settings
|
|
datacenters: list[str] = Field(
|
|
default=["EUR-IS-1"],
|
|
description="Preferred datacenters in priority order",
|
|
)
|
|
cloud_type: Literal["SECURE", "COMMUNITY"] = Field(
|
|
default="SECURE",
|
|
description="Cloud type (SECURE or COMMUNITY)",
|
|
)
|
|
container_disk_gb: int = Field(
|
|
default=50,
|
|
ge=10,
|
|
le=1000,
|
|
description="Container disk size in GB",
|
|
)
|
|
min_vram_gb: int = Field(
|
|
default=16,
|
|
ge=8,
|
|
le=80,
|
|
description="Minimum GPU VRAM in GB",
|
|
)
|
|
|
|
# API Settings
|
|
api_timeout: int = Field(
|
|
default=60,
|
|
ge=10,
|
|
le=300,
|
|
description="API request timeout in seconds",
|
|
)
|
|
max_retries: int = Field(
|
|
default=3,
|
|
ge=0,
|
|
le=10,
|
|
description="Maximum API retry attempts",
|
|
)
|
|
retry_backoff: float = Field(
|
|
default=2.0,
|
|
ge=1.0,
|
|
le=10.0,
|
|
description="Exponential backoff multiplier for retries",
|
|
)
|
|
|
|
# Monitoring Settings
|
|
log_poll_interval: int = Field(
|
|
default=5,
|
|
ge=1,
|
|
le=60,
|
|
description="S3 log polling interval in seconds",
|
|
)
|
|
pod_status_poll_interval: int = Field(
|
|
default=10,
|
|
ge=5,
|
|
le=120,
|
|
description="Pod status polling interval in seconds",
|
|
)
|
|
completion_check_patterns: list[str] = Field(
|
|
default=[
|
|
"Training complete",
|
|
"Model saved to",
|
|
"✓ Training finished",
|
|
"SUCCESS:",
|
|
],
|
|
description="Patterns to detect training completion in logs",
|
|
)
|
|
error_patterns: list[str] = Field(
|
|
default=[
|
|
"CUDA out of memory",
|
|
"RuntimeError:",
|
|
"AssertionError:",
|
|
"FAILED:",
|
|
"ERROR:",
|
|
"panic!",
|
|
],
|
|
description="Patterns to detect errors in logs",
|
|
)
|
|
|
|
# Paths
|
|
log_file_path: str = Field(
|
|
default="logs/training.log",
|
|
description="Path to training log file in volume (relative to mount)",
|
|
)
|
|
models_dir: str = Field(
|
|
default="models",
|
|
description="Path to models directory in volume (relative to mount)",
|
|
)
|
|
|
|
model_config = SettingsConfigDict(
|
|
env_file=".env.runpod",
|
|
env_file_encoding="utf-8",
|
|
case_sensitive=False,
|
|
extra="ignore",
|
|
)
|
|
|
|
@field_validator("runpod_s3_endpoint")
|
|
@classmethod
|
|
def validate_endpoint(cls, v: str) -> str:
|
|
"""Ensure S3 endpoint is a valid HTTPS URL."""
|
|
if not v.startswith("https://"):
|
|
raise ValueError("S3 endpoint must use HTTPS")
|
|
return v
|
|
|
|
@field_validator("datacenters")
|
|
@classmethod
|
|
def validate_datacenters(cls, v: list[str]) -> list[str]:
|
|
"""Ensure at least one datacenter is specified."""
|
|
if not v:
|
|
raise ValueError("At least one datacenter must be specified")
|
|
return v
|
|
|
|
@classmethod
|
|
def load_from_file(cls, env_file: str | Path = ".env.runpod") -> "RunPodConfig":
|
|
"""
|
|
Load configuration from a specific env file.
|
|
|
|
Args:
|
|
env_file: Path to .env file (relative to project root or absolute)
|
|
|
|
Returns:
|
|
Validated RunPodConfig instance
|
|
|
|
Raises:
|
|
ConfigurationError: If configuration is invalid or file not found
|
|
"""
|
|
env_path = Path(env_file)
|
|
|
|
# Try project root if relative path
|
|
if not env_path.is_absolute():
|
|
# We're in foxhunt/runpod/, go up 1 level to project root
|
|
project_root = Path(__file__).parent.parent
|
|
env_path = project_root / env_file
|
|
|
|
if not env_path.exists():
|
|
raise ConfigurationError(
|
|
f"Configuration file not found: {env_path}\n"
|
|
f"Create .env.runpod in project root with required credentials."
|
|
)
|
|
|
|
try:
|
|
# Override model_config to use specific file
|
|
config = cls(_env_file=str(env_path))
|
|
return config
|
|
except Exception as e:
|
|
raise ConfigurationError(f"Failed to load configuration: {e}") from e
|
|
|
|
def get_s3_bucket(self) -> str:
|
|
"""Get S3 bucket name (same as volume ID)."""
|
|
return self.runpod_volume_id
|
|
|
|
def get_log_s3_key(self) -> str:
|
|
"""Get S3 key for training log file."""
|
|
return self.log_file_path
|
|
|
|
def get_models_s3_prefix(self) -> str:
|
|
"""Get S3 prefix for models directory."""
|
|
prefix = self.models_dir
|
|
if not prefix.endswith("/"):
|
|
prefix += "/"
|
|
return prefix
|
|
|
|
def get_api_headers(self) -> dict[str, str]:
|
|
"""Get HTTP headers for RunPod API requests."""
|
|
return {
|
|
"Content-Type": "application/json",
|
|
"Authorization": f"Bearer {self.runpod_api_key}",
|
|
}
|
|
|
|
|
|
# Global config instance (lazy-loaded)
|
|
_config: RunPodConfig | None = None
|
|
|
|
|
|
def get_config(reload: bool = False) -> RunPodConfig:
|
|
"""
|
|
Get global configuration instance.
|
|
|
|
Args:
|
|
reload: Force reload configuration from file
|
|
|
|
Returns:
|
|
RunPodConfig instance
|
|
|
|
Raises:
|
|
ConfigurationError: If configuration cannot be loaded
|
|
"""
|
|
global _config
|
|
|
|
if _config is None or reload:
|
|
_config = RunPodConfig.load_from_file()
|
|
|
|
return _config
|