[Unit]
Description=Foxhunt ML Training Service (Advanced AI/ML Training Pipeline)
After=foxhunt-core.service foxhunt-data.service
Requires=foxhunt-core.service foxhunt-data.service
Documentation=https://github.com/foxhunt/docs
StartLimitInterval=0

[Service]
Type=exec
User=foxhunt
Group=foxhunt
WorkingDirectory=/opt/foxhunt
ExecStart=/opt/foxhunt/bin/foxhunt-ml-training
ExecReload=/bin/kill -HUP $MAINPID
ExecStop=/bin/kill -TERM $MAINPID
TimeoutStartSec=120
TimeoutStopSec=60
Restart=always
RestartSec=15

# Performance Optimizations for ML Training
# GPU-enabled CPU cores with NUMA awareness (cores 10-15 for ML training)
CPUAffinity=10-15
SchedulingPolicy=batch
SchedulingPriority=0
Nice=5

# High memory allocation for large ML models and training datasets
MemoryAccounting=yes
MemoryMax=16G
MemorySwapMax=0

# File and resource limits
LimitNOFILE=131072
LimitMEMLOCK=infinity
LimitCORE=infinity
LimitNPROC=65536

# GPU access for CUDA/ROCm acceleration
DeviceAllow=/dev/nvidia0 rwm
DeviceAllow=/dev/nvidia1 rwm
DeviceAllow=/dev/nvidia-uvm rwm
DeviceAllow=/dev/nvidia-uvm-tools rwm
DeviceAllow=/dev/nvidiactl rwm
DeviceAllow=/dev/nvidia-caps/nvidia-cap1 rwm
DeviceAllow=/dev/nvidia-caps/nvidia-cap2 rwm
SupplementaryGroups=video

# I/O optimizations for large dataset processing
IOSchedulingClass=2
IOSchedulingPriority=4
IOWeight=500

# Security hardening while maintaining GPU access
NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
ReadWritePaths=/opt/foxhunt/data /opt/foxhunt/models /var/log/foxhunt /tmp /dev/shm /opt/foxhunt/checkpoints
ReadOnlyPaths=/opt/foxhunt/config

# Network isolation (ML training should not need external network access)
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
IPAddressAllow=localhost
IPAddressAllow=127.0.0.0/8
IPAddressAllow=::1/128

# Environment variables for ML training
Environment=RUST_LOG=info
Environment=FOXHUNT_CONFIG=/opt/foxhunt/config/production.toml
Environment=FOXHUNT_CORE_ENDPOINT=http://localhost:8080
Environment=FOXHUNT_DATA_ENDPOINT=http://localhost:8084
Environment=FOXHUNT_ENV=production

# CUDA/GPU environment
Environment=CUDA_VISIBLE_DEVICES=0,1
Environment=CUDA_CACHE_PATH=/tmp/cuda-cache
Environment=NVIDIA_DRIVER_CAPABILITIES=compute,utility
Environment=NVIDIA_REQUIRE_CUDA=cuda>=11.8

# ML framework optimizations
Environment=OMP_NUM_THREADS=6
Environment=MKL_NUM_THREADS=6
Environment=PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:512
Environment=TF_GPU_MEMORY_GROWTH=true
Environment=XLA_FLAGS=--xla_gpu_cuda_data_dir=/usr/local/cuda

# Model storage paths
Environment=FOXHUNT_MODEL_STORE=/opt/foxhunt/models
Environment=FOXHUNT_CHECKPOINT_DIR=/opt/foxhunt/checkpoints
Environment=FOXHUNT_TENSORBOARD_DIR=/opt/foxhunt/tensorboard

# Training hyperparameters (can be overridden by config)
Environment=FOXHUNT_BATCH_SIZE=32
Environment=FOXHUNT_LEARNING_RATE=0.001
Environment=FOXHUNT_MAX_EPOCHS=1000

[Install]
WantedBy=multi-user.target
Also=foxhunt-ml-training-canary.service