gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: rename tli→fxt, delete legacy scripts/RunPod/deploy artifacts

Browse Source 
- Rename tli/ directory to fxt/, update package + binary name to "fxt"
- Replace all `use tli::` → `use fxt::` across 52 Rust files
- Update build.rs proto paths (tli/proto → fxt/proto) in 6 services
- Update Dockerfiles, CI workflows, deploy.sh for new paths
- Delete ~170 legacy shell scripts (kept 15 essential ones)
- Delete RunPod Python client (runpod/), tests (tests/runpod/)
- Delete foxhunt-deploy crate (RunPod-only deployment tool)
- Delete terraform/runpod/ (moved to Scaleway)
- Delete ML Python hyperopt scripts (replaced by Rust Argmin PSO)
- Delete .gitlab-ci.yml (using GitHub + Gitea)
- Remove foxhunt-deploy from workspace members

504 files changed, -74,355 lines of legacy code removed.
Workspace compiles clean (0 errors, 0 warnings).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-24 10:31:47 +01:00
parent ff15341c3e
commit 2da5bafc0e
504 changed files with 257 additions and 74355 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
.github/workflows/build.yml vendored
View File

@@ -108,7 +108,7 @@ jobs:
exit 1
fi
build-tli:
build-fxt:
name: Build TLI Client
runs-on: ${{ matrix.os }}
timeout-minutes: 20
@@ -119,16 +119,16 @@ jobs:
include:
- os: ubuntu-latest
target: x86_64-unknown-linux-gnu
artifact_name: tli
asset_name: tli-linux-amd64
artifact_name: fxt
asset_name: fxt-linux-amd64
- os: macos-latest
target: x86_64-apple-darwin
artifact_name: tli
asset_name: tli-macos-amd64
artifact_name: fxt
asset_name: fxt-macos-amd64
- os: windows-latest
target: x86_64-pc-windows-msvc
artifact_name: tli.exe
asset_name: tli-windows-amd64.exe
artifact_name: fxt.exe
asset_name: fxt-windows-amd64.exe
steps:
- name: Checkout Code
@@ -151,7 +151,7 @@ jobs:
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- name: Build TLI
run: cargo build --release -p tli --target ${{ matrix.target }}
run: cargo build --release -p fxt --target ${{ matrix.target }}
- name: Strip Binary (Linux/macOS)
if: matrix.os != 'windows-latest'
@@ -167,7 +167,7 @@ jobs:
create-release:
name: Create GitHub Release
runs-on: ubuntu-latest
needs: [docker-build, build-tli]
needs: [docker-build, build-fxt]
if: startsWith(github.ref, 'refs/tags/v')
timeout-minutes: 10
@@ -252,7 +252,7 @@ jobs:
summary:
name: Build Summary
runs-on: ubuntu-latest
needs: [docker-build, build-tli]
needs: [docker-build, build-fxt]
if: always()
steps:
@@ -263,7 +263,7 @@ jobs:
echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY
echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY
echo "| Docker Build | ${{ needs.docker-build.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| TLI Build | ${{ needs.build-tli.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| FXT Build | ${{ needs.build-fxt.result }} |" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
if [ "${{ github.ref }}" = "refs/heads/main" ]; then

10
.github/workflows/ci-cd-pipeline.yml vendored
View File

@@ -79,7 +79,7 @@ jobs:
path: |
target/release/trading_service
target/release/backtesting_service
target/release/tli
target/release/fxt
retention-days: 30
build-and-test:
@@ -229,7 +229,7 @@ jobs:
with:
images: |
ghcr.io/${{ github.repository }}/foxhunt-trading-engine
ghcr.io/${{ github.repository }}/foxhunt-tli
ghcr.io/${{ github.repository }}/foxhunt-fxt
ghcr.io/${{ github.repository }}/foxhunt-ml
ghcr.io/${{ github.repository }}/foxhunt-risk
ghcr.io/${{ github.repository }}/foxhunt-data
@@ -256,10 +256,10 @@ jobs:
- name: Build and push TLI service
uses: docker/build-push-action@v5
with:
context: ./tli
file: ./tli/Dockerfile.production
context: ./fxt
file: ./fxt/Dockerfile.production
push: true
tags: ghcr.io/${{ github.repository }}/foxhunt-tli:${{ github.sha }}
tags: ghcr.io/${{ github.repository }}/foxhunt-fxt:${{ github.sha }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max

28
.github/workflows/comprehensive-testing.yml vendored
View File

@@ -145,7 +145,7 @@ jobs:
EOF
- name: Build test harness
run: cargo build --bin tli --bin ml-training-service --bin trading-service
run: cargo build --bin fxt --bin ml-training-service --bin trading-service
- name: Run Layer 1 Foundation Tests
run: |
@@ -241,7 +241,7 @@ jobs:
- name: Start services for integration testing
run: |
# Start services in background
cargo run --bin tli -- --config tests/config/tli-test.toml &
cargo run --bin fxt -- --config tests/config/fxt-test.toml &
sleep 5
cargo run --bin ml-training-service -- --config tests/config/ml-test.toml &
sleep 5
@@ -368,8 +368,8 @@ jobs:
- name: Start complete service stack
run: |
# Start all services with proper config
cargo run --bin tli -- --config tests/config/tli-workflow.toml &
TLI_PID=$!
cargo run --bin fxt -- --config tests/config/fxt-workflow.toml &
FXT_PID=$!
sleep 5
cargo run --bin ml-training-service -- --config tests/config/ml-workflow.toml &
@@ -381,7 +381,7 @@ jobs:
sleep 10
# Store PIDs for cleanup
echo $TLI_PID > tli.pid
echo $FXT_PID > fxt.pid
echo $ML_PID > ml.pid
echo $TRADING_PID > trading.pid
@@ -393,7 +393,7 @@ jobs:
- name: Cleanup services
if: always()
run: |
if [ -f tli.pid ]; then kill $(cat tli.pid) || true; fi
if [ -f fxt.pid ]; then kill $(cat fxt.pid) || true; fi
if [ -f ml.pid ]; then kill $(cat ml.pid) || true; fi
if [ -f trading.pid ]; then kill $(cat trading.pid) || true; fi
@@ -521,8 +521,8 @@ jobs:
- name: Start optimized service stack for performance testing
run: |
# Start services with performance-optimized configs
RUST_LOG=warn cargo run --release --bin tli -- --config tests/config/tli-performance.toml &
TLI_PID=$!
RUST_LOG=warn cargo run --release --bin fxt -- --config tests/config/fxt-performance.toml &
FXT_PID=$!
sleep 5
RUST_LOG=warn cargo run --release --bin ml-training-service -- --config tests/config/ml-performance.toml &
@@ -533,7 +533,7 @@ jobs:
TRADING_PID=$!
sleep 10
echo $TLI_PID > tli.pid
echo $FXT_PID > fxt.pid
echo $ML_PID > ml.pid
echo $TRADING_PID > trading.pid
@@ -567,7 +567,7 @@ jobs:
- name: Cleanup performance test services
if: always()
run: |
if [ -f tli.pid ]; then kill $(cat tli.pid) || true; fi
if [ -f fxt.pid ]; then kill $(cat fxt.pid) || true; fi
if [ -f ml.pid ]; then kill $(cat ml.pid) || true; fi
if [ -f trading.pid ]; then kill $(cat trading.pid) || true; fi
@@ -688,8 +688,8 @@ jobs:
- name: Start resilient service stack for chaos testing
run: |
# Start services with resilience-focused configs
cargo run --release --bin tli -- --config tests/config/tli-chaos.toml &
TLI_PID=$!
cargo run --release --bin fxt -- --config tests/config/fxt-chaos.toml &
FXT_PID=$!
sleep 5
cargo run --release --bin ml-training-service -- --config tests/config/ml-chaos.toml &
@@ -700,7 +700,7 @@ jobs:
TRADING_PID=$!
sleep 10
echo $TLI_PID > tli.pid
echo $FXT_PID > fxt.pid
echo $ML_PID > ml.pid
echo $TRADING_PID > trading.pid
@@ -732,7 +732,7 @@ jobs:
- name: Cleanup chaos test services
if: always()
run: |
if [ -f tli.pid ]; then kill $(cat tli.pid) || true; fi
if [ -f fxt.pid ]; then kill $(cat fxt.pid) || true; fi
if [ -f ml.pid ]; then kill $(cat ml.pid) || true; fi
if [ -f trading.pid ]; then kill $(cat trading.pid) || true; fi

454
.gitlab-ci.yml
View File

@@ -1,454 +0,0 @@
# =============================================================================
# GitLab CI/CD Pipeline - Foxhunt Runpod Docker Deployment
# =============================================================================
# Purpose: Automated Docker builds for Runpod GPU deployment
# Registry: Docker Hub (jgrusewski/foxhunt) - PRIVATE repository
# Base Image: Dockerfile.foxhunt-build (CUDA 12.4.1 + cuDNN 9, Ubuntu 22.04)
# Build Strategy: Layer caching with BuildKit
# Versioning: Git commit SHA + timestamp
# Deployment: Manual trigger for production
#
# CRITICAL: Requires Docker Hub authentication via GitLab CI/CD Variables
# =============================================================================
# Pipeline stages
stages:
- build
- test
- deploy
# Global variables
variables:
# Docker configuration
DOCKER_DRIVER: overlay2
DOCKER_BUILDKIT: 1
DOCKER_TLS_CERTDIR: "/certs"
# Fix for GitLab Runner v18.5.0: Override invalid "if-not-available" with correct value
# Valid values: "always", "if-not-present", "never"
# Using "if-not-present" for faster builds with local cache fallback
DOCKER_PULL_POLICY: "if-not-present"
# Image configuration
DOCKER_HUB_REGISTRY: docker.io
DOCKER_HUB_REPO: jgrusewski/foxhunt
IMAGE_TAG: ${DOCKER_HUB_REPO}:${CI_COMMIT_SHORT_SHA}
IMAGE_TAG_LATEST: ${DOCKER_HUB_REPO}:latest
# Build arguments
GIT_COMMIT: ${CI_COMMIT_SHORT_SHA}
BUILD_DATE: ${CI_COMMIT_TIMESTAMP}
# Rust configuration
RUST_BACKTRACE: "1"
RUST_LOG: "info"
# =============================================================================
# REQUIRED GitLab CI/CD Variables (Settings > CI/CD > Variables)
# =============================================================================
# DOCKER_HUB_USERNAME - Docker Hub username (e.g., jgrusewski)
# DOCKER_HUB_PASSWORD - Docker Hub access token (NOT password, use token!)
#
# To create Docker Hub access token:
# 1. Login to hub.docker.com
# 2. Account Settings > Security > New Access Token
# 3. Name: "GitLab CI/CD", Permissions: Read, Write, Delete
# 4. Copy token to GitLab CI/CD variable DOCKER_HUB_PASSWORD
# 5. Mark variable as "Masked" and "Protected" in GitLab
#
# IMPORTANT: Docker Hub repository must be set to PRIVATE
# =============================================================================
# Docker-in-Docker service
services:
- name: docker:24.0.7-dind
# Fix for GitLab Runner v18.5.0: Use valid pull_policy value
# "if-not-present" allows local cache usage while falling back to pull if needed
pull_policy: ["if-not-present"]
# Global before_script: Docker Hub authentication
before_script:
- echo "Authenticating with Docker Hub..."
- echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY
- docker info
# =============================================================================
# BUILD STAGE - Docker Image Build with Layer Caching
# =============================================================================
build:docker:
stage: build
image:
name: docker:24.0.7
pull_policy: ["if-not-present"]
tags:
- docker
before_script:
# Docker Hub authentication
- echo "Authenticating with Docker Hub..."
- echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY
# Set build metadata
- export GIT_COMMIT=$CI_COMMIT_SHORT_SHA
- export BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
- echo "Building Docker image..."
- 'echo " Commit: $GIT_COMMIT"'
- 'echo " Date: $BUILD_DATE"'
- 'echo " Tags: $IMAGE_TAG, $IMAGE_TAG_LATEST"'
script:
# Pull latest image for layer caching
- echo "Pulling latest image for layer caching..."
- docker pull $IMAGE_TAG_LATEST || true
# Build Docker image with BuildKit and layer caching
- |
DOCKER_BUILDKIT=1 docker build \
-f Dockerfile.foxhunt-build \
-t $IMAGE_TAG \
-t $IMAGE_TAG_LATEST \
--build-arg GIT_COMMIT=$GIT_COMMIT \
--build-arg BUILD_DATE=$BUILD_DATE \
--cache-from $IMAGE_TAG_LATEST \
--label "org.opencontainers.image.created=$BUILD_DATE" \
--label "org.opencontainers.image.revision=$GIT_COMMIT" \
--label "org.opencontainers.image.source=$CI_PROJECT_URL" \
--label "org.opencontainers.image.url=$CI_PROJECT_URL" \
--label "org.opencontainers.image.title=Foxhunt HFT Trading System" \
--label "org.opencontainers.image.description=CUDA 12.4.1 + cuDNN 9 runtime for Runpod GPU deployment" \
.
# Push both tags to Docker Hub
- echo "Pushing Docker images to Docker Hub..."
- docker push $IMAGE_TAG
- docker push $IMAGE_TAG_LATEST
# Display image info
- "echo \"Docker image build complete:\""
- docker images | grep jgrusewski/foxhunt
- docker history --no-trunc $IMAGE_TAG | head -10
# Build artifacts (image metadata)
after_script:
- docker inspect $IMAGE_TAG > image-metadata.json || true
artifacts:
name: "docker-image-${CI_COMMIT_SHORT_SHA}"
paths:
- image-metadata.json
expire_in: 30 days
reports:
dotenv: image-metadata.env
# Build only on main branch
rules:
- if: '$CI_COMMIT_BRANCH == "main"'
when: always
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
timeout: 30m
# Retry on infrastructure failures
retry:
max: 2
when:
- runner_system_failure
- stuck_or_timeout_failure
# =============================================================================
# TEST STAGE - GLIBC and CUDA Validation
# =============================================================================
test:glibc-validation:
stage: test
image:
name: docker:24.0.7
pull_policy: ["if-not-present"]
tags:
- docker
before_script:
- echo "Authenticating with Docker Hub..."
- echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY
- docker pull $IMAGE_TAG
script:
- "echo \"Test 1: Verifying GLIBC version...\""
- docker run --rm $IMAGE_TAG ldd --version
- "echo \"Test 2: Verifying GLIBC symbols for hyperopt binaries...\""
- |
for binary in hyperopt_mamba2_demo hyperopt_dqn_demo hyperopt_ppo_demo hyperopt_tft_demo; do
echo "Checking GLIBC symbols for $binary..."
docker run --rm -v /runpod-volume:/runpod-volume $IMAGE_TAG \
sh -c "if [ -f /runpod-volume/binaries/$binary ]; then ldd /runpod-volume/binaries/$binary | grep 'GLIBC_2'; else echo 'Binary not found on volume (expected in CI)'; fi"
done
- "echo \"Test 3: Verifying base system libraries...\""
- docker run --rm $IMAGE_TAG ldconfig -p | grep -E "(libstdc|libgcc_s|libm\.so|libc\.so)"
- "echo \"Test 4: Verifying ca-certificates...\""
- docker run --rm $IMAGE_TAG ls -la /etc/ssl/certs/ | head -5
# Run tests only after successful build
needs:
- build:docker
rules:
- if: '$CI_COMMIT_BRANCH == "main"'
when: always
timeout: 10m
test:cuda-validation:
stage: test
image:
name: docker:24.0.7
pull_policy: ["if-not-present"]
tags:
- docker
before_script:
- echo "Authenticating with Docker Hub..."
- echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY
- docker pull $IMAGE_TAG
script:
- "echo \"Test 1: Verifying CUDA installation...\""
- docker run --rm $IMAGE_TAG ls -la /usr/local/cuda/bin/ | grep nvcc || echo "nvcc not found (expected for runtime-only)"
- "echo \"Test 2: Verifying CUDA libraries...\""
- docker run --rm $IMAGE_TAG ls -la /usr/local/cuda/lib64/ | grep -E "(libcuda|libcurand|libcublas|libcublasLt)"
- "echo \"Test 3: Verifying cuDNN 9 installation...\""
- docker run --rm $IMAGE_TAG ls -la /usr/local/cuda/lib64/ | grep libcudnn || docker run --rm $IMAGE_TAG find /usr -name "libcudnn*" 2>/dev/null
- "echo \"Test 4: Verifying CUDA environment variables...\""
- docker run --rm $IMAGE_TAG env | grep -E "(CUDA_HOME|LD_LIBRARY_PATH|NVIDIA_VISIBLE_DEVICES|CUDA_VISIBLE_DEVICES)"
- "echo \"Test 5: Verifying CUDA compat libraries...\""
- docker run --rm $IMAGE_TAG ls -la /usr/local/cuda/compat/ | grep libcuda || echo "CUDA compat libraries available"
# Run tests only after successful build
needs:
- build:docker
rules:
- if: '$CI_COMMIT_BRANCH == "main"'
when: always
timeout: 10m
test:entrypoint-validation:
stage: test
image:
name: docker:24.0.7
pull_policy: ["if-not-present"]
tags:
- docker
before_script:
- echo "Authenticating with Docker Hub..."
- echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY
- docker pull $IMAGE_TAG
script:
- "echo \"Test 1: Verifying entrypoint scripts...\""
- docker run --rm $IMAGE_TAG ls -la /entrypoint.sh /entrypoint-generic.sh
- "echo \"Test 2: Verifying entrypoint scripts are executable...\""
- docker run --rm $IMAGE_TAG sh -c "test -x /entrypoint.sh && test -x /entrypoint-generic.sh && echo 'Entrypoints are executable'"
- "echo \"Test 3: Running entrypoint help...\""
- docker run --rm $IMAGE_TAG --help || echo "Entrypoint help executed (expected failure without volume)"
# Run tests only after successful build
needs:
- build:docker
rules:
- if: '$CI_COMMIT_BRANCH == "main"'
when: always
timeout: 5m
# =============================================================================
# DEPLOY STAGE - Manual Production Deployment
# =============================================================================
deploy:runpod:
stage: deploy
image:
name: docker:24.0.7
pull_policy: ["if-not-present"]
tags:
- docker
before_script:
- echo "Preparing for Runpod deployment..."
script:
- echo "======================================================================"
- "echo \"Docker image ready for Runpod deployment:\""
- "echo \" Image: $IMAGE_TAG\""
- "echo \" Latest: $IMAGE_TAG_LATEST\""
- "echo \" Commit: $GIT_COMMIT\""
- "echo \" Built: $BUILD_DATE\""
- echo "======================================================================"
- echo ""
- "echo \"Deployment Instructions:\""
- "echo \"1. Login to Runpod.io\""
- "echo \"2. Deploy GPU Pod:\""
- "echo \" - Region: EUR-IS-1 (required for volume mount)\""
- "echo \" - GPU: RTX A4000 (16GB, \\$0.25/hr) or Tesla V100 (16GB, \\$0.10/hr)\""
- "echo \" - Docker Image: $IMAGE_TAG\""
- "echo \" - Container Registry Auth: Select Docker Hub credentials\""
- "echo \" - Volume Mount: /runpod-volume (select existing network volume)\""
- "echo \"3. Pod will auto-start training from /runpod-volume/binaries/\""
- "echo \"4. Models saved to /runpod-volume/models/ (auto-synced to S3)\""
- echo ""
- "echo \"Alternative: Use automated deployment script:\""
- "echo \" python3 scripts/runpod_deploy.py --gpu-type 'RTX A4000' --image $IMAGE_TAG\""
- echo "======================================================================"
# Manual deployment approval required
when: manual
# Deploy only on main branch
rules:
- if: '$CI_COMMIT_BRANCH == "main"'
when: manual
# Require all tests to pass before deployment
needs:
- build:docker
- test:glibc-validation
- test:cuda-validation
- test:entrypoint-validation
# Deployment environment
environment:
name: production/runpod
url: https://www.runpod.io/console/pods
deployment_tier: production
action: start
timeout: 5m
deploy:runpod-staging:
stage: deploy
image:
name: docker:24.0.7
pull_policy: ["if-not-present"]
tags:
- docker
before_script:
- echo "Preparing for Runpod staging deployment..."
script:
- echo "======================================================================"
- "echo \"Docker image ready for Runpod STAGING deployment:\""
- "echo \" Image: $IMAGE_TAG\""
- "echo \" Commit: $GIT_COMMIT\""
- echo "======================================================================"
- echo ""
- "echo \"Staging Deployment (Auto-triggered for testing):\""
- "echo \"1. Deploy to Runpod staging environment\""
- "echo \"2. Run basic smoke tests (1 epoch training)\""
- "echo \"3. Verify model output and metrics\""
- "echo \"4. Validate S3 sync functionality\""
- echo ""
- "echo \"Use automated deployment script with --test flag:\""
- "echo \" python3 scripts/runpod_deploy.py --gpu-type 'Tesla V100' --image $IMAGE_TAG --test\""
- echo "======================================================================"
# Auto-deploy to staging (for testing)
when: on_success
# Deploy to staging on main branch
rules:
- if: '$CI_COMMIT_BRANCH == "main"'
when: on_success
# Require all tests to pass
needs:
- build:docker
- test:glibc-validation
- test:cuda-validation
- test:entrypoint-validation
# Staging environment
environment:
name: staging/runpod
url: https://www.runpod.io/console/pods
deployment_tier: staging
action: start
auto_stop_in: 1 hour
timeout: 5m
# =============================================================================
# CLEANUP - Docker Hub Old Images (Optional)
# =============================================================================
cleanup:docker-hub:
stage: deploy
image:
name: docker:24.0.7
pull_policy: ["if-not-present"]
tags:
- docker
before_script:
- echo "Authenticating with Docker Hub..."
- echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY
script:
- echo "======================================================================"
- "echo \"Docker Hub Cleanup (Manual)\""
- echo "======================================================================"
- "echo \"To clean up old images on Docker Hub:\""
- "echo \"1. Login to hub.docker.com\""
- "echo \"2. Navigate to jgrusewski/foxhunt repository\""
- "echo \"3. Delete old tags (keep last 10-20 commits)\""
- "echo \"4. Keep 'latest' tag for production\""
- echo ""
- "echo \"Alternative: Use Docker Hub API to auto-cleanup\""
- "echo \" (Requires additional scripting - out of scope for this pipeline)\""
- echo "======================================================================"
# Manual cleanup trigger
when: manual
rules:
- if: '$CI_COMMIT_BRANCH == "main"'
when: manual
timeout: 2m
# =============================================================================
# PIPELINE NOTIFICATIONS (Optional)
# =============================================================================
# Uncomment to enable Slack/Discord notifications
# notify:success:
# stage: .post
# script:
# - 'curl -X POST -H "Content-Type: application/json" --data "{\"text\":\"✅ Foxhunt Docker build SUCCESS: $IMAGE_TAG\"}" $SLACK_WEBHOOK_URL'
# when: on_success
# rules:
# - if: '$CI_COMMIT_BRANCH == "main"'
#
# notify:failure:
# stage: .post
# script:
# - 'curl -X POST -H "Content-Type: application/json" --data "{\"text\":\"❌ Foxhunt Docker build FAILED: $CI_PIPELINE_URL\"}" $SLACK_WEBHOOK_URL'
# when: on_failure
# rules:
# - if: '$CI_COMMIT_BRANCH == "main"'
# =============================================================================
# PIPELINE SUMMARY
# =============================================================================
# Stages:
# 1. BUILD (build:docker) - Build Docker image with layer caching
# 2. TEST (test:*) - Validate GLIBC, CUDA, and entrypoint scripts
# 3. DEPLOY (deploy:*) - Manual production deployment, auto staging deployment
#
# Triggers:
# - Push to main branch: Auto-build + auto-test + manual deploy
# - Merge request: Pipeline disabled (no builds on MRs)
#
# Build time: ~2-3 minutes (cached), ~5-8 minutes (clean build)
# Image size: ~4.8GB (CUDA 12.4.1 + cuDNN 9 + Ubuntu 22.04)
# Registry: Docker Hub (jgrusewski/foxhunt) - PRIVATE
#
# Security:
# - Docker Hub credentials stored in GitLab CI/CD Variables (masked)
# - PRIVATE Docker Hub repository (authentication required)
# - Manual deployment approval for production
# - Staging environment with auto-stop (1 hour)
#
# Cost:
# - GitLab CI/CD: Free tier (400 CI/CD minutes/month)
# - Docker Hub: Free tier (1 private repo, unlimited pulls)
# - Runpod GPU: Pay-per-use ($0.10-0.29/hr depending on GPU)
# =============================================================================

140
Cargo.lock generated
View File

@@ -217,6 +217,7 @@ dependencies = [
"criterion",
"dashmap 6.1.0",
"futures",
"fxt",
"governor",
"hdrhistogram",
"hex",
@@ -248,7 +249,6 @@ dependencies = [
"sha2",
"sqlx",
"thiserror 1.0.69",
"tli",
"tokio",
"tokio-stream",
"tokio-test",
@@ -1553,6 +1553,7 @@ dependencies = [
"dbn 0.42.0",
"dotenvy",
"futures",
"fxt",
"influxdb2",
"jsonwebtoken",
"ml",
@@ -1579,7 +1580,6 @@ dependencies = [
"storage",
"tempfile",
"thiserror 1.0.69",
"tli",
"tokio",
"tokio-stream",
"tokio-test",
@@ -3757,6 +3757,7 @@ dependencies = [
"fastrand",
"flate2",
"futures",
"fxt",
"http 1.3.1",
"lazy_static",
"prometheus",
@@ -3773,7 +3774,6 @@ dependencies = [
"sqlx",
"tempfile",
"thiserror 1.0.69",
"tli",
"tokio",
"tokio-stream",
"tonic 0.14.2",
@@ -3783,32 +3783,6 @@ dependencies = [
"uuid",
]
[[package]]
name = "foxhunt-deploy"
version = "1.0.0"
dependencies = [
"assert_cmd",
"aws-config",
"aws-sdk-s3",
"chrono",
"clap",
"colored",
"dotenvy",
"home",
"indicatif",
"predicates",
"regex",
"reqwest 0.12.23",
"serde",
"serde_json",
"tempfile",
"thiserror 1.0.69",
"tokio",
"toml",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "foxhunt_e2e"
version = "0.1.0"
@@ -4007,6 +3981,59 @@ dependencies = [
"slab",
]
[[package]]
name = "fxt"
version = "1.0.0"
dependencies = [
"aes-gcm",
"anyhow",
"argon2",
"assert_cmd",
"async-trait",
"base64 0.22.1",
"chrono",
"clap",
"colored",
"comfy-table",
"common",
"console",
"criterion",
"dirs 5.0.1",
"futures",
"futures-util",
"getrandom 0.2.16",
"hex",
"indicatif",
"jsonwebtoken",
"keyring",
"once_cell",
"owo-colors",
"predicates",
"proptest",
"prost 0.14.1",
"rand 0.8.5",
"rpassword",
"rust_decimal",
"serde",
"serde_json",
"serial_test",
"sha2",
"tabled",
"tempfile",
"thiserror 1.0.69",
"tokio",
"tokio-stream",
"tokio-test",
"toml",
"tonic 0.14.2",
"tonic-prost",
"tonic-prost-build",
"tracing",
"tracing-subscriber",
"uuid",
"zeroize",
]
[[package]]
name = "gemm"
version = "0.18.2"
@@ -9629,6 +9656,7 @@ dependencies = [
"dhat",
"dotenvy",
"futures",
"fxt",
"hdrhistogram",
"influxdb2",
"jemalloc_pprof",
@@ -9654,7 +9682,6 @@ dependencies = [
"sqlx",
"tempfile",
"thiserror 1.0.69",
"tli",
"tokio",
"tokio-stream",
"tokio-test",
@@ -9844,59 +9871,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tli"
version = "1.0.0"
dependencies = [
"aes-gcm",
"anyhow",
"argon2",
"assert_cmd",
"async-trait",
"base64 0.22.1",
"chrono",
"clap",
"colored",
"comfy-table",
"common",
"console",
"criterion",
"dirs 5.0.1",
"futures",
"futures-util",
"getrandom 0.2.16",
"hex",
"indicatif",
"jsonwebtoken",
"keyring",
"once_cell",
"owo-colors",
"predicates",
"proptest",
"prost 0.14.1",
"rand 0.8.5",
"rpassword",
"rust_decimal",
"serde",
"serde_json",
"serial_test",
"sha2",
"tabled",
"tempfile",
"thiserror 1.0.69",
"tokio",
"tokio-stream",
"tokio-test",
"toml",
"tonic 0.14.2",
"tonic-prost",
"tonic-prost-build",
"tracing",
"tracing-subscriber",
"uuid",
"zeroize",
]
[[package]]
name = "tokio"
version = "1.47.1"

7
Cargo.toml
View File

@@ -109,7 +109,7 @@ members = [
"risk",
"risk-data",
"trading-data",
"tli",
"fxt",
"ml",
"ml-data",
"data",
@@ -135,7 +135,6 @@ members = [
"tests",
"tests/e2e",
"tests/load_tests",
"foxhunt-deploy",
"web-gateway",
"ctrader-openapi",
]
@@ -382,7 +381,7 @@ arc-swap = "1.6"
# foxhunt-common-types = { path = "foxhunt-common-types" } # DELETED - types migrated to common/src/types.rs
trading_engine = { path = "trading_engine" }
data = { path = "data" }
tli = { path = "tli" }
fxt = { path = "fxt" }
risk = { path = "risk" }
risk-data = { path = "risk-data" }
backtesting = { path = "backtesting" }
@@ -438,7 +437,7 @@ rand_distr.workspace = true
serde_json.workspace = true
sqlx.workspace = true
tempfile = "3.13"
tli.workspace = true # Required by tests/fixtures/mod.rs
fxt.workspace = true # Required by tests/fixtures/mod.rs
tokio.workspace = true
trading_engine.workspace = true
uuid.workspace = true

2
config/mutants.toml
View File

@@ -10,7 +10,7 @@ minimum_test_score = 80
# Packages to test (focus on critical components)
# We prioritize ML, trading engine, and risk management
exclude_packages = [
"tli", # Pure client - less critical
"fxt", # Pure client - less critical
"data_acquisition_service", # Lower priority
"monitoring_service", # Lower priority
]

6
deploy.sh
View File

@@ -209,7 +209,7 @@ build_images() {
docker build -t foxhunt/backtesting-service:latest -f services/backtesting_service/Dockerfile .
# Build TLI
docker build -t foxhunt/tli:latest -f tli/Dockerfile .
docker build -t foxhunt/fxt:latest -f fxt/Dockerfile .
log_success "Docker images built successfully"
}
@@ -222,7 +222,7 @@ deploy_services() {
trading-service \
ml-training-service \
backtesting-service \
tli
fxt
log_info "Waiting for application services to start..."
sleep 30
@@ -246,7 +246,7 @@ deploy_full() {
health_check() {
log_info "Performing health check..."
local services=("trading-service" "ml-training-service" "backtesting-service" "tli")
local services=("trading-service" "ml-training-service" "backtesting-service" "fxt")
local failed_services=()
for service in "${services[@]}"; do

63
foxhunt-deploy/Cargo.toml
View File

@@ -1,63 +0,0 @@
[package]
name = "foxhunt-deploy"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
homepage.workspace = true
documentation.workspace = true
publish.workspace = true
keywords.workspace = true
categories.workspace = true
[dependencies]
# CLI Framework
clap = { version = "4.5", features = ["derive", "env", "color"] }
# Async Runtime
tokio = { workspace = true }
# HTTP Client
reqwest = { version = "0.12", features = ["json", "rustls-tls"], default-features = false }
# AWS SDK (for S3 log monitoring)
aws-config = "1.5"
aws-sdk-s3 = "1.50"
# Serialization
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
toml = "0.8"
# Error Handling
thiserror = "1.0"
# Terminal UI
indicatif = "0.17"
colored = "2.1"
# Environment Variables
dotenvy = "0.15"
# Logging
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
# Utilities
chrono = { version = "0.4", features = ["serde"] }
home = "0.5"
regex = "1.10"
[dev-dependencies]
assert_cmd = "2.0"
predicates = "3.1"
tempfile = "3.13"
[[bin]]
name = "foxhunt-deploy"
path = "src/main.rs"
[lints]
workspace = true

289
foxhunt-deploy/DEPLOYMENT_QUICK_START.md
View File

@@ -1,289 +0,0 @@
# RunPod Deployment Quick Start Guide
## Prerequisites
1. **Initialize Configuration**:
```bash
foxhunt-deploy init
```
2. **Edit Configuration**:
Edit `~/.runpod/config.toml` and add your RunPod API key:
```toml
[runpod]
api_key = "YOUR_RUNPOD_API_KEY_HERE"
```
## Common Deployment Scenarios
### 1. PPO Training (Production)
Deploy PPO training with dual learning rates:
```bash
foxhunt-deploy deploy \
--command "train_ppo_parquet --policy-lr 1e-6 --value-lr 0.001 --epochs 200" \
--gpu-type "RTX A4000" \
--name "ppo-production"
```
**Expected Cost**: $0.25/hr (~$0.50 for 2 hours)
### 2. DQN Hyperparameter Optimization
Run hyperopt for DQN with 50 trials:
```bash
foxhunt-deploy deploy \
--command "hyperopt_dqn_demo --n-trials 50" \
--gpu-type "RTX A4000" \
--name "dqn-hyperopt"
```
**Expected Cost**: $0.25/hr (~$0.13 for 30 minutes)
### 3. MAMBA-2 Training (High Memory)
Train MAMBA-2 with RTX 4090 for larger batch sizes:
```bash
foxhunt-deploy deploy \
--command "train_mamba2_dbn --batch-size 256" \
--gpu-type "RTX 4090" \
--name "mamba2-batch256"
```
**Expected Cost**: $0.59/hr (~$0.30 for 30 minutes)
### 4. TFT Training (Fast Cache)
Train TFT with cache optimization:
```bash
foxhunt-deploy deploy \
--command "train_tft_parquet --parquet-file /runpod-volume/data/ES_FUT_180d.parquet --epochs 50" \
--gpu-type "RTX A4000" \
--name "tft-fp32"
```
**Expected Cost**: $0.25/hr (~$0.02 for 5 minutes)
### 5. Multi-Model Hyperopt (Long Running)
Run hyperopt for multiple models (TFT + PPO):
```bash
# TFT Hyperopt
foxhunt-deploy deploy \
--command "hyperopt_tft_demo --n-trials 50" \
--gpu-type "RTX A4000" \
--name "tft-hyperopt"
# PPO Hyperopt
foxhunt-deploy deploy \
--command "hyperopt_ppo_demo --n-trials 50" \
--gpu-type "RTX A4000" \
--name "ppo-hyperopt"
```
**Expected Cost**: $0.25/hr each (~$0.25 total for parallel execution)
### 6. Custom Environment Variables
Deploy with custom logging and debugging:
```bash
foxhunt-deploy deploy \
--command "train_ppo_parquet --epochs 100" \
--env "RUST_LOG=debug" \
--env "RUST_BACKTRACE=1" \
--env "CUDA_VISIBLE_DEVICES=0"
```
### 7. Dry Run (Validation)
Test deployment configuration without actually deploying:
```bash
foxhunt-deploy deploy \
--command "train_ppo --epochs 100" \
--gpu-type "RTX 4090" \
--dry-run
```
**Output**:
```
✓ Selected GPU: RTX 4090 (24 GB VRAM, $0.59/hr)
ℹ [DRY RUN] Deployment Summary
────────────────────────────────────────────────────────────
Pod Name: foxhunt-20251102-091802
GPU: RTX 4090
VRAM: 24 GB
Cost: $0.59/hr
Datacenter: EUR-IS-1
Image: jgrusewski/foxhunt:latest
Command: train_ppo --epochs 100
Container Disk: 50 GB
────────────────────────────────────────────────────────────
✓ Dry run completed successfully. Request is valid.
```
## GPU Selection Guide
### RTX A4000 (Default)
- **VRAM**: 16 GB
- **Cost**: $0.25/hr
- **Best For**: Most training jobs (PPO, DQN, TFT)
- **Selection**: Auto-selected or `--gpu-type "RTX A4000"`
### RTX 4090
- **VRAM**: 24 GB
- **Cost**: $0.59/hr
- **Best For**: Large batch sizes (MAMBA-2, TFT with large cache)
- **Selection**: `--gpu-type "RTX 4090"`
### A40
- **VRAM**: 48 GB
- **Cost**: $0.45/hr
- **Best For**: Very large models or ensembles
- **Selection**: `--gpu-type "A40"`
## Datacenter Selection
### EUR-IS-1 (Default)
- **Location**: Europe (Iceland)
- **Latency**: ~50ms from EU
- **Availability**: High
- **Selection**: Auto-selected or `--datacenter "EUR-IS-1"`
### US-TX-3
- **Location**: US (Texas)
- **Latency**: ~30ms from US East
- **Availability**: Medium
- **Selection**: `--datacenter "US-TX-3"`
## Monitoring and Management
### Monitor Pod Logs
After deployment, monitor training progress:
```bash
# Replace <pod_id> with actual pod ID from deployment output
foxhunt-deploy monitor <pod_id>
```
### Terminate Pod
When training completes:
```bash
# Replace <pod_id> with actual pod ID
foxhunt-deploy run terminate --pod-id <pod_id>
```
### Check Last Pod ID
The CLI saves the last deployed pod ID to `.last_pod_id`:
```bash
cat .last_pod_id
```
## Cost Estimates
| Training Job | GPU | Duration | Cost |
|---|---|---|---|
| PPO Production | RTX A4000 | 2 hours | $0.50 |
| DQN Hyperopt | RTX A4000 | 30 min | $0.13 |
| MAMBA-2 Training | RTX 4090 | 30 min | $0.30 |
| TFT Training | RTX A4000 | 5 min | $0.02 |
| PPO Hyperopt | RTX A4000 | 15 min | $0.06 |
**Total for full ML suite**: ~$1.00 (with hyperopt)
## Troubleshooting
### API Key Not Set
**Error**: `RunPod API key not set`
**Solution**:
```bash
# Edit config file
nano ~/.runpod/config.toml
# Add API key
[runpod]
api_key = "YOUR_RUNPOD_API_KEY_HERE"
```
### GPU Not Available
**Error**: `GPU type 'RTX 4090' not found`
**Solution**:
1. Try auto-selection (remove `--gpu-type` flag)
2. Use RTX A4000 instead: `--gpu-type "RTX A4000"`
3. Check RunPod dashboard for availability
### Command Failed
**Error**: `Docker start command cannot be empty`
**Solution**:
Ensure `--command` flag is provided:
```bash
foxhunt-deploy deploy --command "train_ppo --epochs 100"
```
### Invalid Environment Variable
**Error**: `Invalid environment variable format: KEY`
**Solution**:
Use `KEY=VALUE` format:
```bash
foxhunt-deploy deploy --command "train_ppo" --env "RUST_LOG=debug"
```
## Best Practices
1. **Always use dry-run first** for new deployments:
```bash
foxhunt-deploy deploy --command "..." --dry-run
```
2. **Use descriptive pod names** for easy identification:
```bash
--name "ppo-production-v2" instead of auto-generated
```
3. **Monitor costs** using dry-run cost estimates before deploying
4. **Terminate pods immediately** when training completes to avoid charges
5. **Use RTX A4000** for most jobs (best value, sufficient for 99% of workloads)
6. **Set environment variables** for debugging during development:
```bash
--env "RUST_LOG=debug" --env "RUST_BACKTRACE=1"
```
## Next Steps
After successful deployment:
1. **Monitor Logs**: `foxhunt-deploy monitor <pod_id>`
2. **Check Training Progress**: Watch for epoch updates, loss metrics
3. **Download Results**: Results saved to `/runpod-volume/ml_training/`
4. **Terminate Pod**: `foxhunt-deploy run terminate --pod-id <pod_id>`
5. **Retrieve Models**: Access via RunPod S3 or volume download
## Support
- **RunPod Dashboard**: https://www.runpod.io/console/pods
- **Foxhunt Docs**: See `RUNPOD_DEPLOYMENT_IMPLEMENTATION.md`
- **CLI Help**: `foxhunt-deploy deploy --help`

378
foxhunt-deploy/DOCKER_BUILD_MILESTONE2.md
View File

@@ -1,378 +0,0 @@
# Milestone 2: Docker Build Implementation
**Status**: ✅ COMPLETE
**Date**: 2025-11-02
**Time**: ~45 minutes
## Summary
Implemented Docker build and push functionality for the foxhunt-deploy CLI. The implementation uses `std::process::Command` to execute Docker commands with real-time output streaming and visual progress indicators.
## Implementation Details
### Module Structure
```
src/docker/
├── mod.rs # Docker verification utilities
├── build.rs # Docker build implementation
└── push.rs # Docker push implementation
```
### Key Components
#### 1. Docker Module (`src/docker/mod.rs`)
**Functions**:
- `verify_docker_available()` - Validates Docker installation and daemon status
- `image_exists(tag)` - Checks if a Docker image exists locally
**Features**:
- Uses `which docker` to verify Docker is installed
- Checks Docker daemon status via `docker version`
- Provides clear error messages for common issues
#### 2. Build Module (`src/docker/build.rs`)
**Main Types**:
- `DockerBuildOptions` - Builder pattern for build configuration
- `build_image()` - Main build function with streaming output
**Features**:
- Builder pattern API for flexible configuration
- Real-time output streaming with `BufReader`
- Progress spinner using `indicatif` crate
- Validates Dockerfile and build context existence
- Captures and returns image ID after successful build
- Detailed error messages with stderr output
**Options**:
- `tag` - Docker image tag (e.g., "jgrusewski/foxhunt:latest")
- `dockerfile` - Path to Dockerfile (default: "Dockerfile.foxhunt-build")
- `context` - Build context directory (default: ".")
- `no_cache` - Disable Docker build cache
- `build_args` - Additional build arguments (reserved for future use)
#### 3. Push Module (`src/docker/push.rs`)
**Main Functions**:
- `push_image(tag)` - Push image to registry
- `check_registry_auth(registry)` - Verify registry authentication
**Features**:
- Validates image exists before pushing
- Real-time progress streaming
- Progress spinner with upload status
- Special handling for authentication errors
- Clear error messages suggesting `docker login`
### Integration
Updated `src/cli/build.rs` to use the new Docker module:
```rust
pub async fn execute(config: &FoxhuntConfig, args: &BuildArgs) -> Result<()> {
// Step 1: Verify Docker
docker::verify_docker_available()?;
// Step 2: Build image
let full_tag = format!("{}/{}:{}",
config.docker.registry,
config.docker.image_name,
args.tag
);
let build_options = DockerBuildOptions::new(full_tag.clone())
.dockerfile(args.dockerfile.clone())
.context(args.context.clone())
.no_cache(args.no_cache);
let _image_id = docker::build::build_image(&build_options)?;
// Step 3: Push (unless --no-push)
if !args.no_push {
docker::push::push_image(&full_tag)?;
}
Ok(())
}
```
## Testing
### Unit Tests (5 tests)
Located in:
- `src/docker/mod.rs` (2 tests)
- `src/docker/build.rs` (2 tests)
- `src/docker/push.rs` (1 test)
**Coverage**:
- ✅ Builder pattern API
- ✅ Default values
- ✅ Docker availability check
- ✅ Image existence check
- ✅ Registry auth check
### Integration Tests (14 tests)
Located in `tests/docker_integration_tests.rs`
**Coverage**:
- ✅ CLI help output
- ✅ Config validation
- ✅ Command-line arguments
- ✅ Error message format validation
- ✅ Flag recognition (--no-push, --no-cache, etc.)
**Test Results**:
```
running 14 tests
test test_build_args_defaults ... ok
test test_build_command_help ... ok
test test_build_error_handling_daemon_not_running ... ok
test test_build_error_handling_no_docker ... ok
test test_build_options_builder_pattern ... ok
test test_build_requires_config ... ok
test test_build_validates_dockerfile_existence ... ok
test test_build_with_custom_context ... ok
test test_build_with_custom_dockerfile ... ok
test test_build_with_custom_tag ... ok
test test_build_with_no_cache_flag ... ok
test test_build_with_no_push_flag ... ok
test test_push_error_handling_image_not_found ... ok
test test_push_error_handling_no_auth ... ok
test result: ok. 14 passed; 0 failed; 0 ignored
```
## Usage Examples
### Basic Build (with push)
```bash
# Builds and pushes jgrusewski/foxhunt:latest
foxhunt-deploy build
```
### Build with Custom Tag (no push)
```bash
# Builds jgrusewski/foxhunt:v1.0.0 without pushing
foxhunt-deploy build --tag v1.0.0 --no-push
```
### Build with Custom Dockerfile
```bash
# Use a different Dockerfile
foxhunt-deploy build --dockerfile Dockerfile.custom --no-cache
```
### Build with Custom Context
```bash
# Use a different build context
foxhunt-deploy build --context ../parent-dir --tag test
```
## Error Handling
The implementation provides clear, actionable error messages:
### Docker Not Installed
```
Error: Docker is not installed. Please install Docker Desktop or Docker Engine.
```
### Docker Daemon Not Running
```
Error: Docker daemon is not running. Please start Docker Desktop or Docker service.
```
### Dockerfile Not Found
```
Error: Dockerfile not found: Dockerfile.foxhunt-build
```
### Build Context Not Found
```
Error: Build context directory not found: .
```
### Authentication Failed
```
Error: Docker registry authentication failed. Please run 'docker login' first.
```
### Image Doesn't Exist (for push)
```
Error: Image 'jgrusewski/foxhunt:latest' does not exist locally. Build it first.
```
## Visual Feedback
The implementation uses `indicatif` for progress indication:
```
[1/3] Verifying Docker installation...
ℹ Building Docker image: jgrusewski/foxhunt:latest
ℹ Dockerfile: Dockerfile.foxhunt-build
ℹ Context: .
⠋ Starting Docker build...
[2/3] Building Docker image...
⠙ Step 5/12: RUN cargo build --release
✓ Built image: jgrusewski/foxhunt:latest (sha256:abc123...)
[3/3] Pushing to Docker registry...
⠸ Pushing to registry...
✓ Successfully pushed: jgrusewski/foxhunt:latest
✓ Docker build completed successfully!
ℹ Image: jgrusewski/foxhunt:latest
```
## Design Decisions
### 1. std::process::Command vs bollard
**Choice**: `std::process::Command`
**Rationale**:
- Simpler implementation (no additional dependencies)
- Direct access to Docker CLI features
- Easier debugging (can run same commands manually)
- Better compatibility across Docker versions
- Minimal overhead
### 2. Real-time Output Streaming
**Choice**: `BufReader` with line-by-line streaming
**Rationale**:
- User sees build progress immediately
- Can debug build failures in real-time
- Better UX for long builds
- Standard pattern for CLI tools
### 3. Builder Pattern for Options
**Choice**: `DockerBuildOptions` with builder methods
**Rationale**:
- Flexible API for future extensions
- Self-documenting code
- Type-safe configuration
- Optional parameters with defaults
### 4. Progress Indicators
**Choice**: `indicatif::ProgressBar` with spinner
**Rationale**:
- Visual feedback that process is running
- Professional CLI appearance
- Updates with current build step
- Clears on completion (no clutter)
### 5. Error Handling Strategy
**Choice**: Validate early, fail fast, provide context
**Rationale**:
- Check Docker installation before attempting build
- Validate file paths before running commands
- Capture stderr for detailed error messages
- Suggest fixes in error messages
## Configuration Integration
The build command integrates with the config system:
```toml
[docker]
registry = "jgrusewski" # Docker Hub username or registry
image_name = "foxhunt" # Image name
tag = "latest" # Default tag (overridden by --tag)
```
**Tag Resolution**:
```
Full Tag = {config.docker.registry}/{config.docker.image_name}:{args.tag}
Example: jgrusewski/foxhunt:latest
```
## Future Enhancements
While not implemented in Milestone 2, the design supports:
1. **Build Args**: The `build_arg()` method exists but isn't exposed via CLI yet
2. **Multi-platform Builds**: Could add `--platform linux/amd64,linux/arm64`
3. **BuildKit Features**: Could enable BuildKit-specific features
4. **Parallel Builds**: Could build multiple tags simultaneously
5. **Registry Selection**: Could support multiple registries
6. **Image Inspection**: Could show image size, layers, etc.
## Files Changed
### New Files (3)
- `src/docker/mod.rs` - Docker utilities (57 lines)
- `src/docker/build.rs` - Build implementation (194 lines)
- `src/docker/push.rs` - Push implementation (119 lines)
- `tests/docker_integration_tests.rs` - Integration tests (177 lines)
### Modified Files (2)
- `src/main.rs` - Added docker module import
- `src/cli/build.rs` - Replaced stub with real implementation (38 lines)
### Fixed Files (1)
- `src/runpod/deployment.rs` - Fixed ownership issue on line 43
**Total**: ~585 lines of production code + tests
## Compilation Status
```bash
$ cargo build
Compiling foxhunt-deploy v1.0.0
Finished `dev` profile [unoptimized + debuginfo] target(s) in 15.17s
$ cargo test --package foxhunt-deploy docker::
Finished `test` profile [unoptimized] target(s) in 2.24s
Running unittests src/main.rs
running 5 tests
test docker::build::tests::test_build_options_defaults ... ok
test docker::build::tests::test_build_options_builder ... ok
test docker::tests::test_verify_docker_available ... ok
test docker::push::tests::test_check_registry_auth ... ok
test docker::tests::test_image_exists ... ok
test result: ok. 5 passed; 0 failed; 0 ignored
```
## Milestone Completion Checklist
- ✅ Create Docker module structure (mod.rs, build.rs, push.rs)
- ✅ Implement `DockerBuildOptions` with builder pattern
- ✅ Implement `build_image()` with real-time output streaming
- ✅ Implement `push_image()` with progress indication
- ✅ Add Docker daemon verification
- ✅ Update build subcommand to use Docker module
- ✅ Add unit tests (5 tests)
- ✅ Add integration tests (14 tests)
- ✅ Verify all tests pass
- ✅ Handle errors gracefully with actionable messages
- ✅ Use `indicatif` for progress bars
- ✅ Stream Docker output in real-time
- ✅ Support all CLI flags (--tag, --no-push, --no-cache, --dockerfile, --context)
- ✅ Validate Docker installation and image existence
## Next Steps (Milestone 3)
The next milestone should implement RunPod deployment functionality:
1. Complete `src/runpod/deployment.rs` implementation
2. Integrate with `src/cli/deploy.rs`
3. Add RunPod API client functionality
4. Test end-to-end deployment workflow
## Notes
- The implementation prioritizes reliability and error handling
- All error messages are actionable and suggest fixes
- The builder pattern makes it easy to extend functionality
- Real-time output streaming provides immediate feedback
- Tests validate both success and error paths

221
foxhunt-deploy/MILESTONE_1_COMPLETE.md
View File

@@ -1,221 +0,0 @@
# Milestone 1: Core Infrastructure - COMPLETE ✅
**Completion Date**: 2025-11-02
**Duration**: ~25 minutes
**Status**: All tests passing (4/4), binary size: 4.2MB
## Implementation Summary
Successfully implemented all core modules for the foxhunt-deploy Rust CLI:
### 1. Configuration System ✅
**Files Created**:
- `src/config/types.rs` (179 lines) - Configuration structs with serde support
- `src/config/mod.rs` (171 lines) - ConfigManager with TOML loading and env overrides
- `examples/sample_config.toml` - Sample configuration file
**Features**:
- `FoxhuntConfig` main struct with nested configs
- `RunPodConfig` (API key, GPU type, datacenter)
- `DockerConfig` (registry, image, tag)
- `S3Config` (endpoint, bucket, region, poll interval)
- `DeploymentDefaults` (disk size, volume settings)
- Default value functions for all fields
- Environment variable overrides from `.env.runpod`
- Config validation (API key, S3 bucket required)
- Sample config creation at `~/.runpod/config.toml`
### 2. Terminal Utilities ✅
**Files Created**:
- `src/utils/terminal.rs` (43 lines) - Colored terminal output
- `src/utils/mod.rs` (4 lines) - Module exports
**Features**:
- `success(msg)` - Green checkmark ✓
- `error(msg)` - Red X ✗
- `info(msg)` - Blue info icon ℹ
- `warning(msg)` - Yellow warning ⚠
- `step(step, total, msg)` - Numbered progress [1/5]
### 3. CLI Framework ✅
**Files Created**:
- `src/cli/mod.rs` (36 lines) - Main CLI struct with clap
- `src/cli/build.rs` (33 lines) - Build subcommand
- `src/cli/deploy.rs` (42 lines) - Deploy subcommand
- `src/cli/monitor.rs` (28 lines) - Monitor subcommand
- `src/cli/run.rs` (33 lines) - Unified run subcommand
**Features**:
- Global flags: `--verbose`, `--config`
- `init` command - Creates sample config
- `build` command - Docker image building (stub)
- `deploy` command - RunPod deployment (stub)
- `monitor` command - Log monitoring (stub)
- `run` command - Unified workflow (stub)
### 4. Main Entry Point ✅
**File Modified**:
- `src/main.rs` (94 lines) - Complete async runtime with routing
**Features**:
- Tokio async runtime
- Clap CLI parsing
- Tracing-subscriber logging (verbose/info levels)
- Config loading with path override
- Config validation
- Subcommand routing
- Graceful error handling
### 5. Error Handling ✅
**File** (existing):
- `src/error.rs` - Custom error types with thiserror
**Error Types**:
- `ConfigNotFound` - Missing config file
- `MissingConfigField` - Required field validation
- `Config` - General config errors
- Plus Docker, RunPod API, S3, IO, HTTP errors
## Test Results
```bash
$ cargo test --package foxhunt-deploy
running 4 tests
test config::tests::test_default_config ... ok
test config::tests::test_env_override ... ok
test config::tests::test_validation ... ok
test utils::terminal::tests::test_terminal_functions ... ok
test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out
```
## Verified Behavior
### 1. CLI Help
```bash
$ foxhunt-deploy --help
Deploy and manage Foxhunt ML training workloads on RunPod GPU infrastructure.
Supports Docker image building, pod deployment, log monitoring, and unified workflows.
Usage: foxhunt-deploy [OPTIONS] <COMMAND>
Commands:
init Initialize configuration file
build Build Docker image
deploy Deploy pod to RunPod
monitor Monitor pod logs
run Build and deploy in one command
help Print this message or the help of the given subcommand(s)
```
### 2. Init Command
```bash
$ foxhunt-deploy init
ℹ Initializing foxhunt-deploy configuration...
✓ Created sample configuration at: /home/user/.runpod/config.toml
ℹ Please edit the config file and add your RunPod API key.
ℹ You can also override settings with a .env.runpod file or environment variables.
```
### 3. Config Validation
```bash
$ rm ~/.runpod/config.toml
$ foxhunt-deploy build
✗ Error: Config file not found at "/home/user/.runpod/config.toml". Run 'foxhunt-deploy init' to create one.
```
### 4. Subcommand Stubs
All subcommands (build, deploy, monitor, run) execute successfully with placeholder messages indicating they're ready for Milestone 2+ implementation.
### 5. Environment Override
```bash
$ RUNPOD_API_KEY="override" foxhunt-deploy build
# Uses overridden API key
```
## Code Statistics
- **Total Files Created**: 11
- **Total Lines of Code**: 770
- **Test Coverage**: 4 unit tests
- **Binary Size**: 4.2MB (release, stripped)
- **Build Time**: 64 seconds (release)
- **Dependencies**: All from Cargo.toml (clap, tokio, serde, toml, etc.)
## File Structure
```
foxhunt-deploy/
├── src/
│ ├── main.rs (94 lines) - Entry point
│ ├── error.rs (66 lines) - Error types
│ ├── cli/
│ │ ├── mod.rs (36 lines) - CLI framework
│ │ ├── build.rs (33 lines) - Build command
│ │ ├── deploy.rs (42 lines) - Deploy command
│ │ ├── monitor.rs (28 lines) - Monitor command
│ │ └── run.rs (33 lines) - Run command
│ ├── config/
│ │ ├── mod.rs (171 lines) - Config manager
│ │ └── types.rs (179 lines) - Config structs
│ └── utils/
│ ├── mod.rs (4 lines) - Utils exports
│ └── terminal.rs (43 lines) - Terminal output
└── examples/
└── sample_config.toml (52 lines) - Sample config
```
## Next Steps (Milestone 2)
1. **Docker Builder Module** - Implement `src/docker/builder.rs`
- Multi-stage build support
- BuildKit caching
- Registry push
- Progress reporting
2. **Build Command Implementation** - Wire up `cli::build::execute()`
- Call Docker builder
- Handle --no-push, --no-cache flags
- Show build progress
3. **Integration Tests** - Add `tests/integration_tests.rs`
- Test config loading
- Test CLI parsing
- Test error handling
## Dependencies Used
- `clap` 4.5 - CLI framework with derive macros
- `tokio` 1.40 - Async runtime
- `serde` 1.0 - Serialization
- `toml` 0.8 - TOML parsing
- `colored` 2.1 - Terminal colors
- `tracing` 0.1 - Structured logging
- `tracing-subscriber` 0.3 - Log formatting
- `dotenvy` 0.15 - .env file loading
- `home` 0.5 - Home directory detection
- `thiserror` 1.0 - Error derive macros
## Critical Design Decisions
1. **Async-First**: All command handlers are async (prepare for HTTP/S3 operations)
2. **Config Hierarchy**: File → Environment → CLI args (standard precedence)
3. **Error Context**: Rich error messages with actionable suggestions
4. **Stub Pattern**: All subcommands return Ok(()) with placeholder messages
5. **Test Coverage**: Unit tests for config, validation, and terminal utils
6. **Binary Optimization**: Release profile with LTO, strip, single codegen unit
## Known Issues
- None. All functionality working as designed.
## Warnings
- 54 compiler warnings (all "unreachable pub item" - expected for now)
- Can be fixed with `cargo fix` later or ignored (not blocking)
---
**Milestone 1 Status**: ✅ **COMPLETE AND CERTIFIED**
Ready for Milestone 2: Docker Builder Implementation

427
foxhunt-deploy/MILESTONE_3_SUMMARY.md
View File

@@ -1,427 +0,0 @@
# Milestone 3 Implementation Summary
**Project**: foxhunt-deploy RunPod Deployment Functionality
**Status**: ✅ **COMPLETE**
**Date**: 2025-11-02
**Time**: 55 minutes
**Code Quality**: Production-Ready
---
## Deliverables
### 1. RunPod Module Structure ✅
Created complete module hierarchy:
```
src/runpod/
├── mod.rs # Module exports and public API
├── types.rs # 167 lines - API request/response types
├── client.rs # 165 lines - RunPodClient implementation
└── deployment.rs # 238 lines - Deployment logic and helpers
```
**Total**: 570 lines of production code
### 2. RunPod API Types ✅
Implemented comprehensive serde types (`src/runpod/types.rs`):
- ✅ `PodDeploymentRequest` - Complete deployment payload with 16 fields
- ✅ `PodDeploymentResponse` - Deployment result with machine/runtime info
- ✅ `GpuType` - GPU information with pricing and VRAM
- ✅ `MachineInfo` - Machine configuration details
- ✅ `RuntimeInfo` - Datacenter and runtime metadata
- ✅ `ApiError` - Error response parsing
- ✅ `PodTerminationResponse` - Termination result
**Features**:
- Full serde serialization/deserialization
- CamelCase API compatibility
- Optional field support
- Type-safe API contracts
### 3. RunPod Client ✅
Implemented async HTTP client (`src/runpod/client.rs`):
```rust
impl RunPodClient {
pub fn new(api_key: String) -> Result<Self>
pub async fn list_gpus(&self) -> Result<Vec<GpuType>>
pub async fn deploy_pod(&self, request: PodDeploymentRequest) -> Result<PodDeploymentResponse>
pub async fn terminate_pod(&self, pod_id: &str) -> Result<()>
pub async fn get_pod(&self, pod_id: &str) -> Result<PodDeploymentResponse>
}
```
**Features**:
- ✅ API key validation
- ✅ 30-second HTTP timeout
- ✅ Bearer token authentication
- ✅ JSON request/response handling
- ✅ Comprehensive error handling
- ✅ Structured logging (info, debug)
**API Integration**:
- Base URL: `https://rest.runpod.io/v1`
- Endpoints: `/pods`, `/pods/{id}`
- Methods: POST, GET, DELETE
- Content-Type: `application/json`
### 4. Deployment Logic ✅
Implemented helper functions (`src/runpod/deployment.rs`):
```rust
pub fn build_deployment_request() -> Result<PodDeploymentRequest>
pub fn select_best_gpu() -> Result<GpuType>
pub fn validate_deployment() -> Result<()>
```
**GPU Selection Algorithm**:
1. User-specified GPU (exact or fuzzy match)
2. Auto-select RTX A4000 (best value: $0.25/hr)
3. Fallback to cheapest available
**Request Builder**:
- ✅ Command parsing (whitespace split)
- ✅ Environment variable map construction
- ✅ Pod name generation with timestamp
- ✅ Configuration defaults application
- ✅ Docker image name formatting
**Validation**:
- ✅ GPU count validation (>= 1)
- ✅ Container disk validation (>= 10GB)
- ✅ Command validation (non-empty)
### 5. Deploy Subcommand ✅
Updated `src/cli/deploy.rs` with full implementation:
**CLI Arguments**:
```bash
--command <COMMAND> # Required: Docker start command
--gpu-type <GPU_TYPE> # Optional: GPU selection
--datacenter <DATACENTER> # Optional: Datacenter preference
--tag <TAG> # Optional: Docker image tag (default: latest)
--env <KEY=VALUE> # Optional: Environment variables (repeatable)
--name <NAME> # Optional: Pod name
--volume-size <SIZE> # Optional: Volume size in GB
--dry-run # Optional: Validate without deploying
```
**Features**:
- ✅ Interactive deployment confirmation
- ✅ Cost estimates (hourly + daily)
- ✅ Deployment summary display
- ✅ Pod ID persistence (`.last_pod_id`)
- ✅ Next-step guidance (monitor, terminate)
- ✅ Dry-run mode for validation
**User Experience**:
```
ℹ Starting RunPod deployment...
✓ Selected GPU: RTX A4000 (16 GB VRAM, $0.25/hr)
ℹ Deployment Summary
────────────────────────────────────────────────────────────
Pod Name: foxhunt-20251102-091802
GPU: RTX A4000
VRAM: 16 GB
Cost: $0.25/hr
Datacenter: EUR-IS-1
Image: jgrusewski/foxhunt:latest
Command: train_ppo --epochs 100
Container Disk: 50 GB
────────────────────────────────────────────────────────────
Estimated cost: $0.25/hr ($6.00/day)
Deploy pod? [y/N]: y
✓ Pod deployed successfully!
```
### 6. Error Handling ✅
Comprehensive error handling:
**Configuration Errors**:
- ❌ Missing API key → User-friendly message
- ❌ Invalid GPU type → Available options listed
- ❌ Invalid environment variables → Format guidance
**API Errors**:
- ❌ HTTP errors (network, timeout) → Parsed and displayed
- ❌ RunPod API errors (quota, GPU unavailable) → User-friendly
- ❌ JSON parsing errors → Debug info provided
**Validation Errors**:
- ❌ Empty command → Clear error message
- ❌ Invalid GPU count → Minimum value enforced
- ❌ Invalid disk size → Minimum value enforced
### 7. Testing ✅
**Unit Tests** (6 tests in `deployment.rs`):
```
✓ test_parse_command # Command parsing
✓ test_parse_command_empty # Error handling
✓ test_select_best_gpu_preferred # User preference
✓ test_select_best_gpu_auto # Auto-selection
✓ test_validate_deployment # Valid request
✓ test_validate_deployment_invalid_gpu_count # Invalid request
```
**Integration Tests**:
- ✅ Dry-run validation
- ✅ Auto GPU selection
- ✅ Custom GPU selection
- ✅ Environment variables
- ✅ Cost estimation
**Test Results**:
```
Running unittests: 19 passed; 0 failed
Running integration tests: 14 passed; 0 failed
Total: 33 tests passed ✅
```
### 8. Documentation ✅
Created comprehensive documentation:
1. **RUNPOD_DEPLOYMENT_IMPLEMENTATION.md** (525 lines):
- Architecture overview
- API integration details
- Configuration guide
- Usage examples
- Testing documentation
- Future enhancements
2. **DEPLOYMENT_QUICK_START.md** (350 lines):
- Common deployment scenarios
- GPU selection guide
- Cost estimates
- Troubleshooting
- Best practices
3. **Inline Documentation**:
- Detailed rustdoc comments
- Function-level documentation
- Example usage in comments
---
## Technical Specifications
### Performance
- **API Response Time**: <500ms for deployment requests
- **Validation**: <10ms for local validation
- **Binary Size**: 21MB (release build, optimized with LTO)
- **Memory**: <20MB for typical operations
- **Build Time**: 76 seconds (clean build)
### Code Metrics
| Component | Lines | Tests | Coverage |
|---|---|---|---|
| types.rs | 167 | - | N/A (data types) |
| client.rs | 165 | - | Manual validation |
| deployment.rs | 238 | 6 | 100% |
| deploy.rs | 242 | - | E2E tested |
| **Total** | **812** | **6** | **100%** |
### Dependencies
**Added**:
- `reqwest` (already in Cargo.toml) - HTTP client
- `serde` (already in Cargo.toml) - Serialization
- `chrono` (already in Cargo.toml) - Timestamps
**No new dependencies required** ✅
---
## Usage Examples
### Example 1: Basic Deployment
```bash
foxhunt-deploy deploy --command "train_ppo --epochs 100"
```
**Output**:
- Auto-selects RTX A4000
- Generates pod name with timestamp
- Displays deployment summary
- Saves pod ID to `.last_pod_id`
### Example 2: Custom Configuration
```bash
foxhunt-deploy deploy \
--command "hyperopt_dqn_demo --n-trials 50" \
--gpu-type "RTX 4090" \
--datacenter "EUR-IS-1" \
--env "RUST_LOG=debug" \
--name "dqn-hyperopt"
```
### Example 3: Dry Run
```bash
foxhunt-deploy deploy \
--command "train_ppo --epochs 100" \
--dry-run
```
**Output**: Validates request without deploying
---
## API Integration
### Request Example
```json
POST https://rest.runpod.io/v1/pods
Authorization: Bearer <API_KEY>
{
"cloudType": "SECURE",
"computeType": "GPU",
"dataCenterIds": ["EUR-IS-1"],
"gpuTypeIds": ["NVIDIA RTX A4000"],
"gpuCount": 1,
"name": "foxhunt-20251102-091802",
"imageName": "jgrusewski/foxhunt:latest",
"containerDiskInGb": 50,
"networkVolumeId": "se3zdnb5o4",
"volumeMountPath": "/runpod-volume",
"dockerStartCmd": ["train_ppo", "--epochs", "100"],
"containerRegistryAuthId": "cmh3ya1710001jo02vwqtisbf",
"ports": ["8888/http", "22/tcp"],
"interruptible": false
}
```
### Response Example
```json
{
"id": "pod_abc123xyz",
"machine": {
"gpuTypeId": "NVIDIA RTX A4000",
"gpuCount": 1,
"vramGb": 16
},
"runtime": {
"datacenterId": "EUR-IS-1"
},
"costPerHr": 0.25
}
```
---
## Completion Checklist
- ✅ RunPod module structure created
- ✅ API types implemented with serde
- ✅ RunPod client with async HTTP
- ✅ Deployment logic with GPU selection
- ✅ Deploy command with interactive UX
- ✅ Dry-run mode for validation
- ✅ Error handling (config, API, validation)
- ✅ Unit tests (6/6 passing)
- ✅ Integration tests (manual validation)
- ✅ Documentation (2 guides + inline)
- ✅ Code compiles without errors
- ✅ Binary tested and working
---
## Time Breakdown
| Task | Time | Notes |
|---|---|---|
| Module setup | 5 min | Created runpod/ directory structure |
| API types | 10 min | Comprehensive serde types |
| RunPod client | 15 min | Async HTTP with error handling |
| Deployment logic | 12 min | GPU selection + validation |
| Deploy command | 10 min | CLI integration + UX |
| Testing | 8 min | Unit tests + manual validation |
| Documentation | 5 min | Implementation guide + quick start |
| **Total** | **55 min** | **Under 60-minute target** ✅ |
---
## Quality Metrics
### Code Quality
- ✅ Production-ready code
- ✅ Comprehensive error handling
- ✅ Type-safe API integration
- ✅ No unsafe code
- ✅ No unwrap() calls in production paths
- ✅ Structured logging throughout
### User Experience
- ✅ Clear, informative output
- ✅ Cost estimates before deployment
- ✅ Interactive confirmation
- ✅ Next-step guidance
- ✅ Dry-run validation mode
- ✅ Helpful error messages
### Maintainability
- ✅ Modular architecture
- ✅ Comprehensive documentation
- ✅ Test coverage
- ✅ Clear separation of concerns
- ✅ No hardcoded values
- ✅ Configuration-driven
---
## Next Steps (Future Enhancements)
1. **GraphQL Integration**:
- Replace hardcoded GPU list with real-time availability
- Support dynamic GPU type discovery
2. **Pod Monitoring**:
- Implement `monitor` subcommand
- Real-time log streaming
- Training metrics parsing
3. **Advanced Features**:
- Multi-GPU deployment support
- Pod templates and presets
- Cost optimization (spot instances)
- SSH key injection
4. **Testing**:
- Mock RunPod API for integration tests
- End-to-end deployment tests
- Load testing for API client
---
## Conclusion
**Milestone 3 is 100% complete** with production-ready code, comprehensive testing, and extensive documentation. The implementation provides a robust, user-friendly CLI for deploying ML training workloads to RunPod GPU infrastructure.
**Key Achievements**:
- ✅ Full REST API integration
- ✅ Smart GPU selection algorithm
- ✅ Interactive user experience
- ✅ Comprehensive error handling
- ✅ 100% test pass rate
- ✅ Complete documentation
**Ready for**: Production deployment and integration with Foxhunt ML training workflows.

480
foxhunt-deploy/MILESTONE_4_SUMMARY.md
View File

@@ -1,480 +0,0 @@
# Milestone 4: S3 Log Monitoring - Implementation Summary
**Status**: ✅ COMPLETE
**Date**: 2025-11-02
**Duration**: ~45 minutes
**Developer**: AI Assistant
## Overview
Successfully implemented S3 log monitoring functionality for the foxhunt-deploy CLI tool. This allows users to stream real-time training logs from RunPod S3 buckets with colorized output, filtering, and automatic completion detection.
## Deliverables
### 1. Core Implementation
#### Files Created
1. **`src/s3/mod.rs`** (186 lines)
- `S3LogClient` struct with AWS SDK wrapper
- Custom RunPod endpoint configuration
- Methods: list_log_files, download_log, download_log_range, get_log_size, log_exists
2. **`src/s3/monitor.rs`** (246 lines)
- `LogMonitor` struct for real-time log streaming
- Poll-based streaming with configurable interval
- Methods: tail_logs, show_recent_logs, list_logs
- Features: auto-detection, retries, completion detection
3. **`src/s3/parser.rs`** (192 lines)
- Log level detection and colorization
- Training metrics parsing (epoch, loss, LR)
- Completion detection
- Filtering and tail utilities
#### Files Modified
1. **`src/cli/monitor.rs`** (57 lines)
- Replaced stub implementation with real S3 client integration
- Added `--list` flag for listing log files
- Connected to LogMonitor for streaming
2. **`src/main.rs`** (96 lines)
- Added `mod s3` declaration
3. **`Cargo.toml`** (71 lines)
- Added `regex = "1.10"` dependency
### 2. Documentation
1. **`README.md`** (500+ lines)
- Complete CLI documentation
- Usage examples for all commands
- Configuration guide
- Troubleshooting section
2. **`S3_MONITOR_IMPLEMENTATION.md`** (400+ lines)
- Detailed technical implementation
- Architecture overview
- API reference
- Performance characteristics
- Known limitations
3. **`MILESTONE_4_SUMMARY.md`** (this file)
- Implementation summary
- Test results
- Next steps
### 3. Examples
1. **`examples/monitor_demo.sh`** (150+ lines)
- Interactive demo script
- 7 usage examples
- Prerequisite checks
- Common patterns reference
## Features Implemented
### ✅ Core Features
1. **Real-Time Log Streaming**
- Poll-based S3 monitoring (5s interval, configurable)
- Byte-range downloads for efficiency
- Tracks last read position
- Only downloads new content
2. **Colorized Output**
- Green: INFO
- Yellow: WARN
- Red: ERROR
- Cyan: DEBUG
- Dimmed: TRACE
3. **Log Discovery**
- Auto-detects primary log file
- Priority: training.log > stdout > stderr
- Lists all available logs with `--list` flag
4. **Tail Mode**
- Show last N lines: `--tail <N>`
- Works in both follow and snapshot modes
- Combines with follow: `--follow --tail 10`
5. **Filtering**
- Regex pattern matching: `--filter "epoch|loss"`
- Filters before colorization
- Show only errors: `--filter "ERROR|WARN"`
6. **Completion Detection**
- Detects training completion keywords
- Exits gracefully when complete
- Patterns: "training complete", "saved final model", etc.
7. **Error Handling**
- Graceful retries (max 3 attempts)
- Network error recovery
- Missing log file handling
- Clear error messages
### ✅ User Experience
1. **Intuitive CLI**
- Consistent with other commands
- Helpful error messages
- Progress indicators
2. **Configuration**
- TOML-based config file
- Environment variable overrides
- Sensible defaults
3. **Documentation**
- Comprehensive README
- Usage examples
- Interactive demo script
- Troubleshooting guide
## Technical Details
### Architecture
```
S3LogClient (mod.rs)
↓ (manages connection)
AWS SDK S3 Client
↓ (authenticated requests)
RunPod S3 Endpoint (https://s3api-eur-is-1.runpod.io)
↓ (downloads logs)
LogMonitor (monitor.rs)
↓ (streams content)
LogParser (parser.rs)
↓ (colorizes)
Terminal Output
```
### Dependencies
```toml
aws-config = "1.5"
aws-sdk-s3 = "1.50"
regex = "1.10"
colored = "2.1" # Already in Cargo.toml
tokio = "1.40" # Already in Cargo.toml
```
### Configuration
**S3 Config** (`~/.foxhunt/config.toml`):
```toml
[s3]
endpoint = "https://s3api-eur-is-1.runpod.io"
bucket = "se3zdnb5o4"
region = "us-east-1"
poll_interval_secs = 5
```
**Environment Variables**:
```bash
export AWS_ACCESS_KEY_ID="your_key"
export AWS_SECRET_ACCESS_KEY="your_secret"
```
### S3 Path Structure
```
s3://se3zdnb5o4/ml_training/
└── {pod_id}/
├── training_runs/
│ └── {model}/
│ └── run_{timestamp}/
│ ├── logs/
│ │ └── training.log (priority 1)
│ └── checkpoints/
├── stdout (priority 2)
└── stderr (priority 3)
```
## Testing
### Build Tests
```bash
# Debug build
cargo build
# Result: ✅ SUCCESS (11 warnings, 0 errors)
# Release build
cargo build --release
# Result: ✅ SUCCESS (compiled in 36.23s)
```
### Unit Tests
```bash
cargo test --package foxhunt-deploy s3
```
**Tests Implemented**:
1. `test_detect_log_level` - ✅ PASS
2. `test_parse_training_metrics` - ✅ PASS
3. `test_detect_completion` - ✅ PASS
4. `test_get_last_n_lines` - ✅ PASS
### Manual Testing
**Commands Verified**:
```bash
# Help output
foxhunt-deploy monitor --help # ✅ PASS
# List logs
foxhunt-deploy monitor <pod_id> --list # ✅ Ready (requires S3 creds)
# Show tail
foxhunt-deploy monitor <pod_id> --tail 20 # ✅ Ready
# Follow logs
foxhunt-deploy monitor <pod_id> --follow # ✅ Ready
# Filter
foxhunt-deploy monitor <pod_id> --follow --filter "epoch" # ✅ Ready
```
## Performance
### Metrics
| Metric | Value | Notes |
|--------|-------|-------|
| Poll Interval | 5s | Configurable in config |
| Latency | 5-10s | Poll interval + S3 latency |
| Network | Byte-range only | Only downloads new content |
| Memory | Streaming | Doesn't load entire file |
| S3 Costs | ~$0.01/month | Minimal GET/HEAD requests |
### Efficiency
- ✅ **Byte-Range Requests**: Only downloads new content since last poll
- ✅ **HEAD Requests**: Checks file size before downloading
- ✅ **Streaming**: Processes chunks, doesn't buffer entire file
- ✅ **Retry Logic**: 3 retries with exponential backoff
## Usage Examples
### 1. List Available Logs
```bash
foxhunt-deploy monitor dy2bn5ninzaxma --list
```
**Output**:
```
Searching for logs for pod: dy2bn5ninzaxma
────────────────────────────────────────────────────────────────────────────────
3 log file(s):
• ml_training/dy2bn5ninzaxma/training_runs/ppo/run_20251102/logs/training.log
• ml_training/dy2bn5ninzaxma/stdout
• ml_training/dy2bn5ninzaxma/stderr
```
### 2. Show Last 20 Lines
```bash
foxhunt-deploy monitor dy2bn5ninzaxma --tail 20
```
### 3. Follow Logs
```bash
foxhunt-deploy monitor dy2bn5ninzaxma --follow
```
**Output**:
```
Monitoring pod: dy2bn5ninzaxma
Poll interval: 5s
────────────────────────────────────────────────────────────────────────────────
Found log file: ml_training/dy2bn5ninzaxma/training.log
────────────────────────────────────────────────────────────────────────────────
INFO: Starting training... [green]
INFO: Epoch: 1, Loss: 0.345 [green]
WARN: Learning rate decayed [yellow]
...
```
### 4. Filter for Metrics
```bash
foxhunt-deploy monitor dy2bn5ninzaxma --follow --filter "epoch|loss"
```
### 5. Show Errors Only
```bash
foxhunt-deploy monitor dy2bn5ninzaxma --follow --filter "ERROR|WARN"
```
## Known Limitations
1. **Poll Delay**: 5-10s lag due to polling (not true streaming)
- **Reason**: S3 doesn't support WebSockets
- **Mitigation**: Configurable poll interval
2. **S3 Costs**: Frequent HEAD/GET requests
- **Impact**: Minimal (~$0.01/month)
- **Efficiency**: Byte-range requests reduce bandwidth
3. **Regex Errors**: Invalid regex shows all lines
- **Reason**: Fail-open for user convenience
- **Alternative**: Could show error message
4. **Single Log File**: Doesn't merge multiple logs
- **Reason**: Complexity vs. value
- **Workaround**: Use `--list` to see all logs
## Future Enhancements
### High Priority
1. **WebSocket Support**: True real-time streaming (if RunPod adds support)
2. **Log Download**: Save logs to local file
3. **Multi-Pod Monitoring**: Monitor multiple pods simultaneously
### Medium Priority
4. **Metrics Dashboard**: Live plot of loss/epoch curves
5. **Log Aggregation**: Merge stdout + stderr + training.log
6. **Compression**: Support .gz log files
### Low Priority
7. **Pagination**: For very large log files
8. **Log Search**: Full-text search within logs
9. **Export**: Export logs to JSON/CSV
## Integration with Existing Workflow
### Before (Manual Process)
1. Deploy pod via Python script
2. SSH into pod or use RunPod dashboard
3. Tail logs manually: `tail -f /runpod-volume/ml_training/*/logs/training.log`
4. Copy-paste to local terminal
### After (Automated)
1. Deploy pod: `foxhunt-deploy deploy --command "train_ppo --epochs 100"`
2. Monitor logs: `foxhunt-deploy monitor <pod_id> --follow`
3. Filter metrics: `--filter "epoch|loss"`
4. Automatic completion detection
**Time Savings**: ~5 minutes per deployment
## Code Quality
### Metrics
- **Lines of Code**: ~700 (3 new files, 2 modified)
- **Test Coverage**: 4 unit tests
- **Documentation**: 1,000+ lines (README + implementation doc)
- **Examples**: 1 interactive demo script
### Best Practices
- ✅ Error handling with custom error types
- ✅ Async/await throughout
- ✅ Configuration management
- ✅ Modular architecture
- ✅ Comprehensive documentation
- ✅ Type safety (strong typing)
- ✅ Resource cleanup (no leaks)
## Deployment Checklist
### Prerequisites
- [x] AWS credentials set (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`)
- [x] Configuration file created (`~/.foxhunt/config.toml`)
- [x] Binary built (`cargo build --release`)
- [x] Binary in PATH (`/usr/local/bin/foxhunt-deploy`)
### Verification
```bash
# 1. Check binary
foxhunt-deploy --version
# Expected: foxhunt-deploy 1.0.0
# 2. Check config
cat ~/.foxhunt/config.toml
# Expected: Valid TOML with [s3] section
# 3. Check credentials
echo $AWS_ACCESS_KEY_ID
# Expected: Non-empty string
# 4. Test monitor help
foxhunt-deploy monitor --help
# Expected: Help text with options
# 5. Test list (if pod exists)
foxhunt-deploy monitor <pod_id> --list
# Expected: List of log files or "No log files found"
```
## Success Criteria
### ✅ All Met
- [x] Real-time log streaming working
- [x] Colorized output implemented
- [x] Filtering functional
- [x] Tail mode working
- [x] Completion detection active
- [x] Error handling robust
- [x] Documentation complete
- [x] Examples provided
- [x] Build succeeds
- [x] Tests pass
## Lessons Learned
### Technical
1. **AWS SDK**: Custom endpoint requires careful configuration
2. **Byte Ranges**: Essential for efficient streaming
3. **Error Handling**: Fail-open for better UX (e.g., invalid regex)
4. **Polling**: Trade-off between latency and S3 costs
### Process
1. **Modular Design**: Separated concerns (client, monitor, parser)
2. **Documentation First**: README written concurrently with code
3. **Examples**: Interactive demo adds significant value
4. **Testing**: Unit tests caught regex edge cases early
## Conclusion
Successfully delivered S3 log monitoring functionality for foxhunt-deploy CLI in ~45 minutes. The implementation is production-ready with:
- ✅ Clean, modular architecture
- ✅ Comprehensive error handling
- ✅ Extensive documentation
- ✅ Interactive examples
- ✅ Unit tests
- ✅ Performance optimizations
The feature integrates seamlessly with existing deploy/run workflows and provides significant time savings for ML training monitoring.
**Next Steps**:
1. Deploy to production
2. Gather user feedback
3. Implement priority enhancements (WebSocket, multi-pod, download)
---
**Implementation Time**: ~45 minutes
**Code Quality**: Production-ready
**User Experience**: Excellent
**Documentation**: Comprehensive
**Status**: ✅ READY FOR PRODUCTION

99
foxhunt-deploy/MONITOR_QUICK_REF.md
View File

@@ -1,99 +0,0 @@
# S3 Monitor Quick Reference
## Prerequisites
```bash
export AWS_ACCESS_KEY_ID="your_runpod_access_key"
export AWS_SECRET_ACCESS_KEY="your_runpod_secret_key"
```
## Common Commands
### List Logs
```bash
foxhunt-deploy monitor <pod_id> --list
```
### Quick Check (last 20 lines)
```bash
foxhunt-deploy monitor <pod_id> --tail 20
```
### Follow in Real-Time
```bash
foxhunt-deploy monitor <pod_id> --follow
```
### Follow with Context
```bash
foxhunt-deploy monitor <pod_id> --follow --tail 10
```
### Filter for Metrics
```bash
foxhunt-deploy monitor <pod_id> --follow --filter "epoch|loss|accuracy"
```
### Show Errors Only
```bash
foxhunt-deploy monitor <pod_id> --follow --filter "ERROR|WARN|CRITICAL"
```
## Config
Location: `~/.foxhunt/config.toml`
```toml
[s3]
endpoint = "https://s3api-eur-is-1.runpod.io"
bucket = "se3zdnb5o4"
region = "us-east-1"
poll_interval_secs = 5 # Decrease for faster updates
```
## Colorization
- 🟢 INFO (green)
- 🟡 WARN (yellow)
- 🔴 ERROR (red)
- 🔵 DEBUG (cyan)
- ⚫ TRACE (dimmed)
## Troubleshooting
### No logs found
```bash
# Check credentials
echo $AWS_ACCESS_KEY_ID
# List logs
foxhunt-deploy monitor <pod_id> --list
# Verify pod ID in RunPod dashboard
```
### Permission denied
```bash
# Check S3 config
cat ~/.foxhunt/config.toml | grep -A 4 "\[s3\]"
# Verify credentials match RunPod account
```
### Slow updates
```bash
# Edit config: reduce poll_interval_secs
nano ~/.foxhunt/config.toml
# Change: poll_interval_secs = 2
# Or check network
ping s3api-eur-is-1.runpod.io
```
## Tips
1. **Ctrl+C** to exit follow mode
2. **Completion auto-detected**: Exits when training finishes
3. **Regex patterns**: Use standard regex syntax
4. **Multiple flags**: Combine `--follow --tail 10 --filter "epoch"`
5. **Log priority**: training.log > stdout > stderr (auto-selected)

302
foxhunt-deploy/QUICK_START_BUILD.md
View File

@@ -1,302 +0,0 @@
# Quick Start: Docker Build Command
## Prerequisites
1. **Docker installed and running**
```bash
docker --version
docker info # Verify daemon is running
```
2. **foxhunt-deploy configured**
```bash
foxhunt-deploy init
# Edit ~/.runpod/config.toml
```
3. **Docker login** (for pushing)
```bash
docker login
```
## Basic Usage
### Build and Push (Default)
```bash
foxhunt-deploy build
```
- Builds: `jgrusewski/foxhunt:latest` (from config)
- Pushes to Docker Hub
- Uses: `Dockerfile.foxhunt-build`
### Build Only (No Push)
```bash
foxhunt-deploy build --no-push
```
- Builds locally
- Skips registry push
- Useful for testing
### Custom Tag
```bash
foxhunt-deploy build --tag v1.0.0
```
- Builds: `jgrusewski/foxhunt:v1.0.0`
- Pushes to Docker Hub
### No Cache Build
```bash
foxhunt-deploy build --no-cache
```
- Rebuilds all layers
- Ignores Docker cache
- Useful after dependency changes
### Custom Dockerfile
```bash
foxhunt-deploy build --dockerfile Dockerfile.custom
```
- Uses different Dockerfile
- Keeps same build context
### Custom Build Context
```bash
foxhunt-deploy build --context ../parent-dir
```
- Uses different directory as context
- Useful for monorepos
## Common Scenarios
### Development Build
```bash
# Quick local test without pushing
foxhunt-deploy build --tag dev --no-push
```
### Production Build
```bash
# Clean build with version tag
foxhunt-deploy build --tag v1.2.3 --no-cache
```
### Test Build
```bash
# Build from different Dockerfile without pushing
foxhunt-deploy build --dockerfile Dockerfile.test --tag test --no-push
```
### Multi-Platform Build (Future)
```bash
# Not yet implemented, but architecture supports it
# foxhunt-deploy build --platform linux/amd64,linux/arm64
```
## Configuration
Edit `~/.runpod/config.toml`:
```toml
[docker]
registry = "jgrusewski" # Your Docker Hub username
image_name = "foxhunt" # Image name
tag = "latest" # Default tag
```
**Result**: Full tag = `jgrusewski/foxhunt:latest`
Override tag with: `--tag your-tag`
## Troubleshooting
### Docker Not Found
```bash
✗ Error: Docker is not installed. Please install Docker Desktop or Docker Engine.
```
**Fix**: Install Docker from https://docker.com
### Docker Daemon Not Running
```bash
✗ Error: Docker daemon is not running. Please start Docker Desktop or Docker service.
```
**Fix**: Start Docker Desktop or `sudo systemctl start docker`
### Dockerfile Not Found
```bash
✗ Error: Dockerfile not found: Dockerfile.foxhunt-build
```
**Fix**: Create Dockerfile or use `--dockerfile` with correct path
### Authentication Failed
```bash
✗ Error: Docker registry authentication failed. Please run 'docker login' first.
```
**Fix**: Run `docker login` and enter credentials
### Permission Denied
```bash
✗ Error: permission denied while trying to connect to Docker daemon
```
**Fix**: Add user to docker group: `sudo usermod -aG docker $USER`
## Environment Variables
Override config with environment variables:
```bash
export DOCKER_REGISTRY=myregistry.io
export DOCKER_IMAGE_NAME=my-app
foxhunt-deploy build --tag v1.0.0
```
## Output Examples
### Successful Build
```
[1/3] Verifying Docker installation...
ℹ Building Docker image: jgrusewski/foxhunt:latest
ℹ Dockerfile: Dockerfile.foxhunt-build
ℹ Context: .
[2/3] Building Docker image...
⠋ Step 5/12: RUN cargo build --release
✓ Built image: jgrusewski/foxhunt:latest (sha256:abc123...)
[3/3] Pushing to Docker registry...
⠸ Pushing to registry...
✓ Successfully pushed: jgrusewski/foxhunt:latest
✓ Docker build completed successfully!
ℹ Image: jgrusewski/foxhunt:latest
```
### Build Failed
```
[1/3] Verifying Docker installation...
[2/3] Building Docker image...
✗ Error: Docker build failed with exit code 1:
ERROR [5/12] RUN cargo build --release
error: package `serde v1.0.228` cannot be built because it requires...
```
### No Push
```
[1/3] Verifying Docker installation...
[2/3] Building Docker image...
✓ Built image: jgrusewski/foxhunt:dev (sha256:xyz789...)
ℹ Skipping push (--no-push flag set)
✓ Docker build completed successfully!
ℹ Image: jgrusewski/foxhunt:dev
```
## Advanced Usage
### Verbose Logging
```bash
foxhunt-deploy build --verbose
```
- Shows debug logs
- Useful for troubleshooting
### Custom Config File
```bash
foxhunt-deploy build --config ./my-config.toml
```
- Uses different config file
- Useful for multiple environments
### Combined Options
```bash
foxhunt-deploy build \
--tag v1.0.0-rc1 \
--dockerfile Dockerfile.prod \
--no-cache \
--verbose
```
## Integration with CI/CD
### GitHub Actions
```yaml
- name: Build Docker Image
run: |
foxhunt-deploy build --tag ${{ github.sha }}
```
### GitLab CI
```yaml
build:
script:
- foxhunt-deploy build --tag ${CI_COMMIT_SHA}
```
### Jenkins
```groovy
stage('Build') {
steps {
sh 'foxhunt-deploy build --tag ${GIT_COMMIT}'
}
}
```
## Performance Tips
1. **Use BuildKit** (Docker 18.09+)
```bash
export DOCKER_BUILDKIT=1
foxhunt-deploy build
```
2. **Leverage Layer Caching**
- Don't use `--no-cache` unless necessary
- Order Dockerfile for optimal caching
- Copy dependency files before source code
3. **Multi-Stage Builds**
- Use Dockerfile.foxhunt-build pattern
- Separate build and runtime stages
- Keep final image small
4. **Parallel Builds**
- Build multiple tags in parallel (manual)
```bash
foxhunt-deploy build --tag v1.0.0 &
foxhunt-deploy build --tag latest &
wait
```
## Next Steps
After building, you can:
1. **Deploy to RunPod**
```bash
foxhunt-deploy deploy --gpu "RTX A4000"
```
2. **Run locally**
```bash
docker run -it jgrusewski/foxhunt:latest bash
```
3. **Inspect image**
```bash
docker images jgrusewski/foxhunt
docker history jgrusewski/foxhunt:latest
```
4. **Test image**
```bash
docker run --rm jgrusewski/foxhunt:latest --version
```
## Help
```bash
foxhunt-deploy build --help
foxhunt-deploy --help
```
## Support
- 📖 Documentation: `DOCKER_BUILD_MILESTONE2.md`
- 📋 Summary: `IMPLEMENTATION_SUMMARY.md`
- 🐛 Issues: Check stderr output with `--verbose`
- 💬 Questions: Review help text and config file

420
foxhunt-deploy/README.md
View File

@@ -1,420 +0,0 @@
# foxhunt-deploy
Rust CLI tool for managing Foxhunt ML training deployments on RunPod GPU infrastructure.
## Features
- 🐳 **Docker Management**: Build and push Docker images to registry
- ☁️ **RunPod Deployment**: Deploy training jobs to GPU pods with one command
- 📊 **S3 Log Monitoring**: Stream real-time logs from RunPod S3 buckets
- 🎨 **Colorized Output**: Easy-to-read log levels (INFO, WARN, ERROR)
- 🔍 **Filtering**: Regex-based log filtering
- ⚙️ **Configuration**: Flexible TOML-based configuration
## Installation
```bash
cd foxhunt-deploy
cargo build --release
# Copy to PATH
sudo cp target/release/foxhunt-deploy /usr/local/bin/
```
## Quick Start
### 1. Initialize Configuration
```bash
foxhunt-deploy init
```
This creates `~/.foxhunt/config.toml` with default settings.
### 2. Configure Credentials
Edit `~/.foxhunt/config.toml`:
```toml
[runpod]
api_key = "YOUR_RUNPOD_API_KEY"
default_gpu_type = "RTX A4000"
default_datacenter = "EUR-IS-1"
[docker]
registry = "jgrusewski"
image_name = "foxhunt"
tag = "latest"
[s3]
endpoint = "https://s3api-eur-is-1.runpod.io"
bucket = "se3zdnb5o4"
region = "us-east-1"
poll_interval_secs = 5
```
Set S3 credentials:
```bash
export AWS_ACCESS_KEY_ID="your_runpod_access_key"
export AWS_SECRET_ACCESS_KEY="your_runpod_secret_key"
```
### 3. Deploy a Training Job
```bash
foxhunt-deploy deploy \
--command "train_ppo --epochs 100" \
--gpu "RTX A4000" \
--datacenter "EUR-IS-1"
```
### 4. Monitor Logs
```bash
# List available log files
foxhunt-deploy monitor <pod_id> --list
# Show last 50 lines
foxhunt-deploy monitor <pod_id> --tail 50
# Follow logs in real-time
foxhunt-deploy monitor <pod_id> --follow
# Filter for specific patterns
foxhunt-deploy monitor <pod_id> --follow --filter "epoch|loss"
```
## Commands
### `init`
Initialize configuration file.
```bash
foxhunt-deploy init
```
Creates `~/.foxhunt/config.toml` with default settings.
### `build`
Build Docker image.
```bash
foxhunt-deploy build [OPTIONS]
Options:
--tag <TAG> Docker image tag [default: latest]
--push Push to registry after building
--dockerfile <PATH> Path to Dockerfile [default: Dockerfile.foxhunt-build]
```
**Example**:
```bash
# Build and push
foxhunt-deploy build --tag v1.0.0 --push
```
### `deploy`
Deploy training job to RunPod GPU.
```bash
foxhunt-deploy deploy [OPTIONS] --command <COMMAND>
Required:
--command <COMMAND> Training command to run
Options:
--name <NAME> Pod name (auto-generated if not provided)
--gpu <GPU> GPU type [default: RTX A4000]
--datacenter <DC> Datacenter [default: EUR-IS-1]
--tag <TAG> Docker image tag [default: latest]
--env <KEY=VALUE> Environment variables (can be repeated)
--volume-size <GB> Network volume size in GB [default: 100]
```
**Examples**:
```bash
# Deploy PPO training
foxhunt-deploy deploy \
--command "train_ppo --epochs 100" \
--gpu "RTX A4000"
# Deploy with custom env vars
foxhunt-deploy deploy \
--command "train_dqn --data /data/ES_FUT.parquet" \
--env "RUST_LOG=debug" \
--env "CUDA_VISIBLE_DEVICES=0"
# Deploy to specific datacenter
foxhunt-deploy deploy \
--command "train_tft --epochs 50" \
--datacenter "US-TX-1" \
--gpu "RTX 4090"
```
### `monitor`
Monitor pod logs from S3.
```bash
foxhunt-deploy monitor [OPTIONS] <POD_ID>
Arguments:
<POD_ID> Pod ID to monitor
Options:
-f, --follow Follow logs in real-time
-t, --tail <N> Number of recent lines to show
-l, --list List available log files
--filter <PATTERN> Filter logs by regex pattern
```
**Examples**:
```bash
# List available log files
foxhunt-deploy monitor dy2bn5ninzaxma --list
# Show last 20 lines
foxhunt-deploy monitor dy2bn5ninzaxma --tail 20
# Follow logs in real-time
foxhunt-deploy monitor dy2bn5ninzaxma --follow
# Follow with initial tail (show last 10 lines, then follow)
foxhunt-deploy monitor dy2bn5ninzaxma --follow --tail 10
# Filter for training metrics
foxhunt-deploy monitor dy2bn5ninzaxma --follow --filter "epoch.*loss"
# Filter for errors only
foxhunt-deploy monitor dy2bn5ninzaxma --follow --filter "ERROR|WARN"
```
### `run`
Execute commands in a running pod.
```bash
foxhunt-deploy run <POD_ID> <COMMAND>
Arguments:
<POD_ID> Pod ID
<COMMAND> Command to execute
```
**Example**:
```bash
foxhunt-deploy run dy2bn5ninzaxma "nvidia-smi"
```
## Configuration
### Config File
Location: `~/.foxhunt/config.toml`
```toml
[runpod]
api_key = "YOUR_API_KEY"
default_gpu_type = "RTX A4000"
default_datacenter = "EUR-IS-1"
[docker]
registry = "jgrusewski"
image_name = "foxhunt"
tag = "latest"
[s3]
endpoint = "https://s3api-eur-is-1.runpod.io"
bucket = "se3zdnb5o4"
region = "us-east-1"
poll_interval_secs = 5
[defaults]
container_disk_gb = 50
volume_path = "/runpod-volume"
volume_size_gb = 100
support_public_ip = false
```
### Environment Variables
Override config values with environment variables:
```bash
# RunPod
export RUNPOD_API_KEY="your_key"
# S3 (required for monitor command)
export AWS_ACCESS_KEY_ID="your_access_key"
export AWS_SECRET_ACCESS_KEY="your_secret_key"
# Optional overrides
export S3_ENDPOINT="https://s3api-eur-is-1.runpod.io"
export S3_BUCKET="se3zdnb5o4"
export S3_POLL_INTERVAL="10"
```
### GPU Types
Common GPU options:
- `RTX A4000` - 16GB VRAM, $0.25/hr (recommended)
- `RTX 4090` - 24GB VRAM, $0.59/hr
- `A40` - 48GB VRAM, $0.79/hr
- `A100 80GB` - 80GB VRAM, $1.89/hr
### Datacenters
Available datacenters:
- `EUR-IS-1` - Europe (Iceland)
- `US-TX-1` - US (Texas)
- `US-OR-1` - US (Oregon)
## S3 Log Monitoring
### How It Works
1. **Log Discovery**: Automatically finds log files in S3 bucket
- Priority: `training.log` > `stdout` > `stderr`
- Path: `s3://{bucket}/ml_training/{pod_id}/...`
2. **Real-Time Streaming**: Polls S3 every 5 seconds (configurable)
- Tracks last read position
- Only downloads new content (byte-range requests)
3. **Colorization**: Applies colors based on log level
- 🟢 INFO (green)
- 🟡 WARN (yellow)
- 🔴 ERROR (red)
- 🔵 DEBUG (cyan)
- ⚫ TRACE (dimmed)
4. **Completion Detection**: Exits when training completes
- Detects keywords: "training complete", "saved final model", etc.
### S3 Path Structure
```
s3://se3zdnb5o4/ml_training/
└── {pod_id}/
├── training_runs/
│ └── {model}/
│ └── run_{timestamp}/
│ ├── logs/
│ │ └── training.log (priority 1)
│ └── checkpoints/
├── stdout (priority 2)
└── stderr (priority 3)
```
### Performance
- **Poll Interval**: 5 seconds (configurable)
- **Latency**: ~5-10s lag
- **Network**: Byte-range downloads (efficient)
- **Memory**: Streams chunks, doesn't load entire file
## Examples
See `examples/monitor_demo.sh` for interactive demo:
```bash
./examples/monitor_demo.sh <pod_id>
```
## Development
### Build
```bash
cargo build --release
```
### Run Tests
```bash
cargo test
```
### Run with Verbose Logging
```bash
foxhunt-deploy --verbose monitor <pod_id> --follow
```
## Troubleshooting
### Monitor: No logs appearing
**Cause**: Missing AWS credentials or incorrect pod ID
**Solution**:
1. Check credentials: `echo $AWS_ACCESS_KEY_ID`
2. List log files: `foxhunt-deploy monitor <pod_id> --list`
3. Verify pod ID in RunPod dashboard
### Monitor: Permission denied
**Cause**: Invalid AWS credentials
**Solution**:
1. Verify RunPod API key has S3 access
2. Check bucket name in config matches RunPod
3. Ensure credentials match RunPod account
### Monitor: Slow updates
**Cause**: High poll interval or network latency
**Solution**:
1. Reduce poll interval in config: `poll_interval_secs = 2`
2. Check network: `ping s3api-eur-is-1.runpod.io`
### Deploy: GPU not available
**Cause**: Requested GPU type unavailable in datacenter
**Solution**:
1. Try different datacenter: `--datacenter "US-TX-1"`
2. Try different GPU: `--gpu "RTX 4090"`
3. Check RunPod dashboard for availability
## Architecture
```
foxhunt-deploy
├── src/
│ ├── cli/ # Command-line interface
│ │ ├── build.rs # Docker build command
│ │ ├── deploy.rs # RunPod deployment command
│ │ ├── monitor.rs # S3 log monitoring command
│ │ └── run.rs # Pod execution command
│ ├── config/ # Configuration management
│ ├── docker/ # Docker operations
│ ├── runpod/ # RunPod API client
│ ├── s3/ # S3 log monitoring
│ │ ├── mod.rs # S3LogClient
│ │ ├── monitor.rs # LogMonitor
│ │ └── parser.rs # Log parsing/colorization
│ ├── utils/ # Utilities (terminal output)
│ └── error.rs # Error handling
└── examples/
└── monitor_demo.sh # Interactive demo
```
## License
Apache-2.0
## Credits
Built for the Foxhunt HFT trading system.

359
foxhunt-deploy/RUNPOD_DEPLOYMENT_IMPLEMENTATION.md
View File

@@ -1,359 +0,0 @@
# RunPod Deployment Implementation - Milestone 3
## Overview
This document describes the implementation of the RunPod deployment functionality for the `foxhunt-deploy` CLI tool. The implementation enables automated deployment of ML training workloads to RunPod GPU infrastructure via REST API integration.
## Architecture
### Module Structure
```
src/runpod/
├── mod.rs # Module exports and public API
├── types.rs # Serde types for RunPod REST API
├── client.rs # RunPodClient with HTTP operations
└── deployment.rs # Deployment logic and helpers
```
### Key Components
#### 1. RunPod API Types (`types.rs`)
Defines serde-compatible types for RunPod REST API communication:
- **`PodDeploymentRequest`**: Complete deployment payload
- Cloud type, compute type, datacenter preferences
- GPU configuration (type, count, VRAM)
- Docker image and startup command
- Volume mounts and network configuration
- Environment variables
- **`PodDeploymentResponse`**: Deployment result
- Pod ID (for monitoring and termination)
- Machine info (GPU type, VRAM)
- Runtime info (datacenter, cost)
- **`GpuType`**: Available GPU information
- Display name, VRAM capacity
- Pricing (secure cloud vs community cloud)
#### 2. RunPod Client (`client.rs`)
HTTP client for RunPod REST API operations:
```rust
impl RunPodClient {
pub fn new(api_key: String) -> Result<Self>
pub async fn list_gpus(&self) -> Result<Vec<GpuType>>
pub async fn deploy_pod(&self, request: PodDeploymentRequest) -> Result<PodDeploymentResponse>
pub async fn terminate_pod(&self, pod_id: &str) -> Result<()>
pub async fn get_pod(&self, pod_id: &str) -> Result<PodDeploymentResponse>
}
```
**API Endpoints**:
- Base URL: `https://rest.runpod.io/v1`
- Authentication: Bearer token (`Authorization: Bearer <API_KEY>`)
- Content-Type: `application/json`
**Error Handling**:
- HTTP status validation
- API error response parsing
- User-friendly error messages
#### 3. Deployment Logic (`deployment.rs`)
Helper functions for deployment operations:
```rust
pub fn build_deployment_request(
args: &DeployArgs,
config: &FoxhuntConfig,
gpu: &GpuType,
) -> Result<PodDeploymentRequest>
pub fn select_best_gpu(
gpus: &[GpuType],
preferred_type: Option<&str>
) -> Result<GpuType>
pub fn validate_deployment(
request: &PodDeploymentRequest
) -> Result<()>
```
**GPU Selection Logic**:
1. User-specified GPU type (exact or fuzzy match)
2. Auto-select RTX A4000 (best value: $0.25/hr)
3. Fallback to cheapest available GPU
**Request Builder**:
- Parses command string into Docker start command array
- Builds environment variable map from CLI args
- Generates pod name with timestamp if not provided
- Applies configuration defaults for volumes, disks, etc.
#### 4. Deploy Command (`cli/deploy.rs`)
User-facing CLI command implementation:
```bash
foxhunt-deploy deploy \
--command "train_ppo --epochs 100" \
--gpu-type "RTX A4000" \
--datacenter "EUR-IS-1" \
--env "CUDA_VISIBLE_DEVICES=0" \
--dry-run
```
**Features**:
- Interactive deployment confirmation with cost estimates
- Dry-run mode for validation without deployment
- Deployment summary display (GPU, cost, datacenter, image, command)
- Pod ID persistence (saved to `.last_pod_id` for reference)
- Next-step guidance (monitoring and termination commands)
## Implementation Details
### API Request Format
Example deployment request to RunPod REST API:
```json
POST https://rest.runpod.io/v1/pods
Authorization: Bearer <API_KEY>
Content-Type: application/json
{
"cloudType": "SECURE",
"computeType": "GPU",
"dataCenterIds": ["EUR-IS-1"],
"gpuTypeIds": ["NVIDIA RTX A4000"],
"gpuCount": 1,
"name": "foxhunt-20251102-091802",
"imageName": "jgrusewski/foxhunt:latest",
"containerDiskInGb": 50,
"networkVolumeId": "se3zdnb5o4",
"volumeMountPath": "/runpod-volume",
"dockerStartCmd": ["train_ppo", "--epochs", "100"],
"containerRegistryAuthId": "cmh3ya1710001jo02vwqtisbf",
"ports": ["8888/http", "22/tcp"],
"interruptible": false,
"env": {
"CUDA_VISIBLE_DEVICES": "0",
"RUST_LOG": "debug"
},
"supportPublicIp": false
}
```
### Configuration
The deployment uses settings from `foxhunt.toml`:
```toml
[runpod]
api_key = "YOUR_RUNPOD_API_KEY"
default_gpu_type = "RTX A4000"
default_datacenter = "EUR-IS-1"
[docker]
registry = "jgrusewski"
image_name = "foxhunt"
tag = "latest"
[defaults]
container_disk_gb = 50
volume_path = "/runpod-volume"
volume_size_gb = 100
support_public_ip = false
```
### Error Handling
Comprehensive error handling for:
1. **Configuration Errors**:
- Missing or invalid API key
- Invalid GPU type selection
2. **API Errors**:
- HTTP errors (network, timeout)
- RunPod API errors (GPU not available, quota exceeded)
- JSON parsing errors
3. **Validation Errors**:
- Invalid command format
- Invalid environment variable format
- Invalid GPU count or disk size
### Security
- API key validation before making requests
- API key stored in config file (gitignored)
- Support for environment variable overrides
- No hardcoded credentials
## Usage Examples
### Basic Deployment
Auto-select GPU and deploy with default settings:
```bash
foxhunt-deploy deploy --command "train_ppo --epochs 100"
```
### Custom GPU Selection
Specify GPU type explicitly:
```bash
foxhunt-deploy deploy \
--command "hyperopt_dqn_demo --n-trials 50" \
--gpu-type "RTX 4090"
```
### Full Configuration
All available options:
```bash
foxhunt-deploy deploy \
--command "train_ppo --epochs 100" \
--gpu-type "RTX A4000" \
--datacenter "EUR-IS-1" \
--tag "v1.2.3" \
--name "ppo-production-run" \
--env "CUDA_VISIBLE_DEVICES=0" \
--env "RUST_LOG=debug" \
--volume-size 150
```
### Dry Run Mode
Validate deployment request without actually deploying:
```bash
foxhunt-deploy deploy \
--command "train_ppo --epochs 100" \
--dry-run
```
Output:
```
ℹ Starting RunPod deployment...
✓ Selected GPU: RTX A4000 (16 GB VRAM, $0.25/hr)
ℹ [DRY RUN] Deployment Summary
────────────────────────────────────────────────────────────
Pod Name: foxhunt-20251102-091802
GPU: RTX A4000
VRAM: 16 GB
Cost: $0.25/hr
Datacenter: EUR-IS-1
Image: jgrusewski/foxhunt:latest
Command: train_ppo --epochs 100
Container Disk: 50 GB
────────────────────────────────────────────────────────────
✓ Dry run completed successfully. Request is valid.
```
## Testing
### Unit Tests
Comprehensive test coverage in `deployment.rs`:
```rust
#[test]
fn test_parse_command() // Command parsing
fn test_parse_command_empty() // Error handling
fn test_select_best_gpu_preferred() // User preference
fn test_select_best_gpu_auto() // Auto-selection
fn test_validate_deployment() // Valid request
fn test_validate_deployment_invalid_gpu_count() // Invalid request
```
**Test Results**:
```
running 19 tests
test runpod::deployment::tests::test_parse_command ... ok
test runpod::deployment::tests::test_parse_command_empty ... ok
test runpod::deployment::tests::test_select_best_gpu_auto ... ok
test runpod::deployment::tests::test_select_best_gpu_preferred ... ok
test runpod::deployment::tests::test_validate_deployment ... ok
test runpod::deployment::tests::test_validate_deployment_invalid_gpu_count ... ok
test result: ok. 19 passed; 0 failed; 0 ignored
```
### Integration Testing
Test with dry-run mode:
```bash
# Test auto GPU selection
foxhunt-deploy deploy --command "train_ppo" --dry-run
# Test custom GPU
foxhunt-deploy deploy --command "train_ppo" --gpu-type "RTX 4090" --dry-run
# Test environment variables
foxhunt-deploy deploy --command "train_ppo" --env "KEY=VALUE" --dry-run
```
## Performance
- **API Response Time**: <500ms for deployment requests
- **Validation**: <10ms for local validation
- **Binary Size**: ~14MB (release build, optimized)
- **Memory**: <20MB for typical operations
## Future Enhancements
1. **GraphQL API Integration**:
- Replace hardcoded GPU list with real-time availability query
- Support more GPU types dynamically
2. **Pod Status Monitoring**:
- Real-time pod status updates
- Automatic retry on transient failures
3. **Cost Optimization**:
- Interruptible instance support
- Multi-datacenter price comparison
- Automatic spot instance selection
4. **Advanced Features**:
- Multi-GPU deployment support
- Custom network volumes
- SSH key injection
- Pod templates and presets
## References
- **RunPod REST API**: https://docs.runpod.io/api/rest
- **RunPod GraphQL API**: https://docs.runpod.io/api/graphql
- **Foxhunt Docker Image**: jgrusewski/foxhunt:latest
- **Private Registry Auth**: cmh3ya1710001jo02vwqtisbf
## Completion Status
- ✅ RunPod module structure created
- ✅ API types implemented with serde
- ✅ RunPod client with async HTTP operations
- ✅ Deployment logic with GPU selection
- ✅ Deploy command with interactive confirmation
- ✅ Dry-run mode for validation
- ✅ Unit tests (6/6 passing)
- ✅ Integration tests (manual validation)
- ✅ Documentation and examples
**Milestone 3: COMPLETE** ✅
**Time Spent**: ~55 minutes
**Code Quality**: Production-ready with comprehensive error handling
**Test Coverage**: 100% for deployment logic

398
foxhunt-deploy/S3_MONITOR_IMPLEMENTATION.md
View File

@@ -1,398 +0,0 @@
# S3 Log Monitoring Implementation
**Status**: ✅ COMPLETE
**Date**: 2025-11-02
**Milestone**: 4 - Monitor Subcommand
## Overview
Implemented real-time S3 log monitoring functionality for the foxhunt-deploy CLI. This allows users to stream training logs from RunPod S3 buckets in real-time with colorized output, filtering, and automatic completion detection.
## Architecture
### Module Structure
```
src/s3/
├── mod.rs # S3LogClient - AWS SDK wrapper
├── monitor.rs # LogMonitor - Log streaming logic
└── parser.rs # Log parsing and colorization utilities
```
### Components
#### 1. S3LogClient (`src/s3/mod.rs`)
AWS SDK wrapper with custom RunPod endpoint support:
```rust
pub struct S3LogClient {
client: aws_sdk_s3::Client,
bucket: String,
endpoint: String,
}
```
**Methods**:
- `new(config: &S3Config)` - Create client with RunPod credentials
- `list_log_files(pod_id: &str)` - List all log files for a pod
- `download_log(path: &str)` - Download complete log file
- `download_log_range(path, start, end)` - Download byte range (for tailing)
- `get_log_size(path: &str)` - Get current file size
- `log_exists(path: &str)` - Check if log file exists
**Configuration**:
- Endpoint: `https://s3api-eur-is-1.runpod.io`
- Bucket: `se3zdnb5o4`
- Region: `us-east-1`
- Credentials: From `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` env vars
#### 2. LogMonitor (`src/s3/monitor.rs`)
Real-time log streaming with polling:
```rust
pub struct LogMonitor {
client: S3LogClient,
pod_id: String,
poll_interval: Duration,
}
```
**Methods**:
- `new(client, pod_id, poll_interval_secs)` - Create monitor
- `tail_logs(tail, filter)` - Stream logs in real-time (follow mode)
- `show_recent_logs(tail)` - Show recent logs (snapshot mode)
- `list_logs()` - List all available log files
**Features**:
- ✅ Automatic log file detection (priority: training.log > stdout > stderr)
- ✅ Real-time streaming with configurable poll interval (default: 5s)
- ✅ Tail mode: show last N lines before following
- ✅ Regex filtering
- ✅ Automatic completion detection
- ✅ Graceful error handling with retries (max 3)
- ✅ Tracks last read position to avoid re-reading
#### 3. LogParser (`src/s3/parser.rs`)
Log parsing and colorization utilities:
```rust
pub enum LogLevel {
Info, // Green
Warn, // Yellow
Error, // Red
Debug, // Cyan
Trace, // Dimmed
Unknown, // Default
}
```
**Functions**:
- `colorize_log_line(line)` - Apply colors based on log level
- `parse_training_metrics(line)` - Extract epoch, loss, LR
- `detect_completion(line)` - Identify training completion
- `filter_line(line, pattern)` - Regex filtering
- `get_last_n_lines(content, n)` - Get tail lines
## Usage
### Prerequisites
1. **AWS Credentials**: Set environment variables
```bash
export AWS_ACCESS_KEY_ID="your_runpod_access_key"
export AWS_SECRET_ACCESS_KEY="your_runpod_secret_key"
```
2. **Configuration**: Create or update `~/.foxhunt/config.toml`
```toml
[s3]
endpoint = "https://s3api-eur-is-1.runpod.io"
bucket = "se3zdnb5o4"
region = "us-east-1"
poll_interval_secs = 5
```
### Examples
#### 1. Follow Logs in Real-Time
```bash
foxhunt-deploy monitor dy2bn5ninzaxma --follow
```
**Output**:
```
Monitoring pod: dy2bn5ninzaxma
Poll interval: 5s
────────────────────────────────────────────────────────────────────────────────
Found log file: ml_training/dy2bn5ninzaxma/training_runs/ppo/run_20251102/logs/training.log
────────────────────────────────────────────────────────────────────────────────
INFO: Starting training... [green]
INFO: Epoch: 1, Loss: 0.345 [green]
WARN: Learning rate decayed [yellow]
INFO: Epoch: 2, Loss: 0.298 [green]
ERROR: CUDA OOM detected [red]
...
────────────────────────────────────────────────────────────────────────────────
✓ Training completed! [green]
```
#### 2. Show Last 50 Lines
```bash
foxhunt-deploy monitor dy2bn5ninzaxma --tail 50
```
#### 3. Follow with Initial Tail
```bash
foxhunt-deploy monitor dy2bn5ninzaxma --follow --tail 20
```
#### 4. Filter by Pattern
```bash
# Show only lines containing "epoch" or "loss"
foxhunt-deploy monitor dy2bn5ninzaxma --follow --filter "epoch.*loss"
# Show only errors
foxhunt-deploy monitor dy2bn5ninzaxma --follow --filter "ERROR|WARN"
```
#### 5. List Available Log Files
```bash
foxhunt-deploy monitor dy2bn5ninzaxma --list
```
**Output**:
```
Searching for logs for pod: dy2bn5ninzaxma
────────────────────────────────────────────────────────────────────────────────
3 log file(s):
• ml_training/dy2bn5ninzaxma/training_runs/ppo/run_20251102/logs/training.log
• ml_training/dy2bn5ninzaxma/stdout
• ml_training/dy2bn5ninzaxma/stderr
```
## S3 Path Structure
```
s3://se3zdnb5o4/ml_training/
├── {pod_id}/
│ ├── training_runs/
│ │ ├── {model}/
│ │ │ ├── run_{timestamp}/
│ │ │ │ ├── logs/
│ │ │ │ │ ├── training.log (priority 1)
│ │ │ │ ├── checkpoints/
│ ├── stdout (priority 2)
│ └── stderr (priority 3)
```
## Features
### ✅ Implemented
1. **Real-Time Streaming**
- Polls S3 every 5 seconds (configurable)
- Tracks last read position
- Only downloads new content (byte-range requests)
2. **Colorized Output**
- Green: INFO
- Yellow: WARN
- Red: ERROR
- Cyan: DEBUG
- Dimmed: TRACE
3. **Filtering**
- Regex pattern matching
- Filter applied before colorization
4. **Tail Mode**
- Show last N lines before following
- Works in both follow and snapshot modes
5. **Completion Detection**
- Detects training completion keywords
- Exits gracefully when training completes
6. **Error Handling**
- Graceful retries (max 3)
- Network error recovery
- Missing log file handling
7. **Multiple Log Files**
- Auto-selects primary log file
- Prioritizes training.log > stdout > stderr
- List mode shows all available logs
### Performance
- **Poll Interval**: 5 seconds (configurable)
- **Network Efficiency**: Byte-range downloads (only new content)
- **Memory**: Streams chunks, doesn't load entire file
- **Latency**: ~5-10s lag (poll interval + S3 latency)
## Configuration
### Default Settings
```toml
[s3]
endpoint = "https://s3api-eur-is-1.runpod.io"
bucket = "se3zdnb5o4"
region = "us-east-1"
poll_interval_secs = 5
```
### Environment Variables
```bash
# Required
export AWS_ACCESS_KEY_ID="your_access_key"
export AWS_SECRET_ACCESS_KEY="your_secret_key"
# Optional (override config)
export S3_ENDPOINT="https://s3api-eur-is-1.runpod.io"
export S3_BUCKET="se3zdnb5o4"
export S3_POLL_INTERVAL="10" # seconds
```
## Dependencies Added
```toml
[dependencies]
aws-config = "1.5"
aws-sdk-s3 = "1.50"
regex = "1.10"
colored = "2.1"
```
## Error Handling
### Network Errors
```rust
Error: S3 error: Failed to get object: network timeout
(retrying 1/3...)
```
- Retries up to 3 times
- 5-second delay between retries
- Fails gracefully if all retries exhausted
### Missing Log Files
```
No log files found yet (retrying in 5s...)
```
- Polls until log file appears
- Useful for newly created pods
- Ctrl+C to exit
### Invalid Credentials
```
Error: Configuration error: AWS_ACCESS_KEY_ID not set
```
- Clear error messages
- Points to missing env vars
## Testing
### Unit Tests
```bash
cargo test --package foxhunt-deploy s3
```
**Tests**:
- `test_detect_log_level` - Log level detection
- `test_parse_training_metrics` - Metric extraction
- `test_detect_completion` - Completion detection
- `test_get_last_n_lines` - Tail functionality
### Integration Test (Manual)
1. Deploy a pod with training job
2. Monitor logs:
```bash
foxhunt-deploy monitor <pod_id> --follow
```
3. Verify:
- ✅ Logs appear in real-time
- ✅ Colors applied correctly
- ✅ Completion detected
- ✅ Graceful exit
## Known Limitations
1. **Poll Delay**: 5-10s lag due to polling (not true streaming)
2. **S3 Costs**: Frequent HEAD/GET requests (minimal, ~$0.01/month)
3. **Regex Filtering**: Invalid regex shows all lines (no error)
4. **Byte Range**: S3 must support byte-range requests (RunPod does)
## Future Enhancements
### Potential Improvements
1. **WebSocket Support**: True real-time streaming (if RunPod adds support)
2. **Log Aggregation**: Merge stdout + stderr + training.log
3. **Metrics Dashboard**: Live plot of loss/epoch curves
4. **Download Mode**: Save logs to local file
5. **Multi-Pod Monitoring**: Monitor multiple pods simultaneously
6. **Compression**: Support .gz log files
7. **Pagination**: For very large log files
### Not Implemented (Out of Scope)
- ❌ SSH log streaming (requires SSH keys)
- ❌ RunPod CLI integration (use their API)
- ❌ Log search/indexing (use grep/less locally)
## Troubleshooting
### Problem: No logs appearing
**Solution**:
1. Check AWS credentials: `echo $AWS_ACCESS_KEY_ID`
2. Verify pod ID: `foxhunt-deploy monitor <pod_id> --list`
3. Check S3 bucket: `aws s3 ls s3://se3zdnb5o4/ --profile runpod`
### Problem: Permission denied
**Solution**:
1. Verify RunPod API key has S3 access
2. Check bucket name in config
3. Ensure credentials match RunPod account
### Problem: Slow updates
**Solution**:
1. Reduce poll interval: `poll_interval_secs = 2` (in config)
2. Check network latency: `ping s3api-eur-is-1.runpod.io`
## Summary
Successfully implemented S3 log monitoring with:
- ✅ Real-time streaming (5s poll interval)
- ✅ Colorized output (5 log levels)
- ✅ Regex filtering
- ✅ Tail mode
- ✅ Completion detection
- ✅ Graceful error handling
- ✅ Production-ready code (tested, documented)
**Time**: ~45 minutes
**Lines of Code**: ~700 (3 files)
**Dependencies**: 3 added (aws-config, aws-sdk-s3, regex)
**Tests**: 4 unit tests
The implementation provides a solid foundation for monitoring RunPod training jobs via S3 logs, with room for future enhancements.

123
foxhunt-deploy/examples/monitor_demo.sh
View File

@@ -1,123 +0,0 @@
#!/bin/bash
# S3 Log Monitor Demo - Usage examples for foxhunt-deploy monitor command
set -e
echo "=== S3 Log Monitor Demo ==="
echo ""
# Check prerequisites
if [[ -z "$AWS_ACCESS_KEY_ID" ]]; then
echo "ERROR: AWS_ACCESS_KEY_ID not set"
echo "Please set RunPod S3 credentials:"
echo " export AWS_ACCESS_KEY_ID='your_key'"
echo " export AWS_SECRET_ACCESS_KEY='your_secret'"
exit 1
fi
# Example pod ID (replace with your actual pod ID)
POD_ID="${1:-dy2bn5ninzaxma}"
echo "Pod ID: $POD_ID"
echo ""
# Function to run example
run_example() {
local description="$1"
local command="$2"
echo "────────────────────────────────────────────────────────────────"
echo "Example: $description"
echo "Command: $command"
echo "────────────────────────────────────────────────────────────────"
echo ""
echo "Press Enter to run (or Ctrl+C to skip)..."
read -r
eval "$command"
echo ""
}
# Example 1: List available log files
run_example \
"List all log files for the pod" \
"foxhunt-deploy monitor $POD_ID --list"
# Example 2: Show last 20 lines
run_example \
"Show last 20 lines of logs" \
"foxhunt-deploy monitor $POD_ID --tail 20"
# Example 3: Show last 50 lines
run_example \
"Show last 50 lines of logs" \
"foxhunt-deploy monitor $POD_ID --tail 50"
# Example 4: Follow logs in real-time
echo "────────────────────────────────────────────────────────────────"
echo "Example: Follow logs in real-time (Ctrl+C to exit)"
echo "Command: foxhunt-deploy monitor $POD_ID --follow"
echo "────────────────────────────────────────────────────────────────"
echo ""
echo "This will stream logs continuously. Press Ctrl+C to stop."
echo "Press Enter to start..."
read -r
foxhunt-deploy monitor "$POD_ID" --follow
# Example 5: Follow with initial tail
echo ""
echo "────────────────────────────────────────────────────────────────"
echo "Example: Follow logs with initial 10 lines"
echo "Command: foxhunt-deploy monitor $POD_ID --follow --tail 10"
echo "────────────────────────────────────────────────────────────────"
echo ""
echo "Press Enter to start (Ctrl+C to stop)..."
read -r
foxhunt-deploy monitor "$POD_ID" --follow --tail 10
# Example 6: Filter by pattern
echo ""
echo "────────────────────────────────────────────────────────────────"
echo "Example: Filter logs for 'epoch' or 'loss'"
echo "Command: foxhunt-deploy monitor $POD_ID --follow --filter 'epoch|loss'"
echo "────────────────────────────────────────────────────────────────"
echo ""
echo "Press Enter to start (Ctrl+C to stop)..."
read -r
foxhunt-deploy monitor "$POD_ID" --follow --filter "epoch|loss"
# Example 7: Filter for errors only
echo ""
echo "────────────────────────────────────────────────────────────────"
echo "Example: Show only errors and warnings"
echo "Command: foxhunt-deploy monitor $POD_ID --follow --filter 'ERROR|WARN'"
echo "────────────────────────────────────────────────────────────────"
echo ""
echo "Press Enter to start (Ctrl+C to stop)..."
read -r
foxhunt-deploy monitor "$POD_ID" --follow --filter "ERROR|WARN"
echo ""
echo "=== Demo Complete ==="
echo ""
echo "Common usage patterns:"
echo ""
echo "1. Quick check: Show last 20 lines"
echo " foxhunt-deploy monitor $POD_ID --tail 20"
echo ""
echo "2. Monitor training: Follow in real-time"
echo " foxhunt-deploy monitor $POD_ID --follow"
echo ""
echo "3. Debug errors: Filter for problems"
echo " foxhunt-deploy monitor $POD_ID --follow --filter 'ERROR|WARN|CRITICAL'"
echo ""
echo "4. Track metrics: Filter for training stats"
echo " foxhunt-deploy monitor $POD_ID --follow --filter 'epoch.*loss|accuracy'"
echo ""
echo "5. List logs: See all available log files"
echo " foxhunt-deploy monitor $POD_ID --list"
echo ""

50
foxhunt-deploy/examples/sample_config.toml
View File

@@ -1,50 +0,0 @@
# Foxhunt RunPod Deployment Configuration
# Edit this file and save to ~/.runpod/config.toml
# You can override any value with environment variables (see .env.runpod example)
[runpod]
# Your RunPod API key (REQUIRED)
# Get it from: https://www.runpod.io/console/user/settings
api_key = "YOUR_RUNPOD_API_KEY_HERE"
# Default GPU type for deployments
default_gpu_type = "RTX A4000"
# Default datacenter region
default_datacenter = "EUR-IS-1"
[docker]
# Docker registry username
registry = "jgrusewski"
# Docker image name
image_name = "foxhunt"
# Default image tag
tag = "latest"
[s3]
# RunPod S3 endpoint (region-specific)
endpoint = "https://s3api-eur-is-1.runpod.io"
# S3 bucket for training outputs
bucket = "se3zdnb5o4"
# AWS region for S3
region = "us-east-1"
# Log polling interval in seconds
poll_interval_secs = 5
[defaults]
# Container disk size in GB
container_disk_gb = 50
# Volume mount path inside container
volume_path = "/runpod-volume"
# Volume size in GB
volume_size_gb = 100
# Whether to enable public IP
support_public_ip = false

67
foxhunt-deploy/src/cli/build.rs
View File

@@ -1,67 +0,0 @@
use crate::config::types::FoxhuntConfig;
use crate::docker;
use crate::error::Result;
use crate::utils;
use clap::Args;
/// Build Docker image with embedded binaries
#[derive(Args, Debug)]
pub(crate) struct BuildArgs {
/// Docker image tag (e.g., "latest", "v1.0.0")
#[arg(short, long, default_value = "latest")]
pub tag: String,
/// Skip pushing to registry after build
#[arg(long)]
pub no_push: bool,
/// Docker build context path
#[arg(long, default_value = ".")]
pub context: String,
/// Dockerfile path
#[arg(long, default_value = "Dockerfile.foxhunt-build")]
pub dockerfile: String,
/// Disable Docker build cache
#[arg(long)]
pub no_cache: bool,
}
/// Execute build command
pub(crate) async fn execute(config: &FoxhuntConfig, args: &BuildArgs) -> Result<()> {
utils::step(1, 3, "Verifying Docker installation...");
docker::verify_docker_available()?;
// Build full image tag from config and args
let full_tag = format!(
"{}/{}:{}",
config.docker.registry,
config.docker.image_name,
args.tag
);
utils::step(2, 3, "Building Docker image...");
// Create build options
let build_options = docker::build::DockerBuildOptions::new(full_tag.clone())
.dockerfile(args.dockerfile.clone())
.context(args.context.clone())
.no_cache(args.no_cache);
// Build the image
let _image_id = docker::build::build_image(&build_options)?;
// Push unless --no-push is specified
if !args.no_push {
utils::step(3, 3, "Pushing to Docker registry...");
docker::push::push_image(&full_tag)?;
} else {
utils::info("Skipping push (--no-push flag set)");
}
utils::success("Docker build completed successfully!");
utils::info(&format!("Image: {}", full_tag));
Ok(())
}

249
foxhunt-deploy/src/cli/deploy.rs
View File

@@ -1,249 +0,0 @@
use crate::config::types::FoxhuntConfig;
use crate::error::Result;
use crate::runpod::{
build_deployment_request, select_best_gpu, validate_deployment, RunPodClient,
};
use crate::utils;
use clap::Args;
use colored::Colorize;
use tracing::info;
/// Deploy pod to RunPod
#[derive(Args, Debug)]
pub(crate) struct DeployArgs {
/// GPU type (e.g., "RTX A4000", "RTX 4090")
#[arg(short, long)]
pub gpu_type: Option<String>,
/// Datacenter region (e.g., "EUR-IS-1", "US-TX-3")
#[arg(short, long)]
pub datacenter: Option<String>,
/// Docker image tag to deploy
#[arg(short, long, default_value = "latest")]
pub tag: String,
/// Training command to run
#[arg(short, long, required = true)]
pub command: String,
/// Environment variables (format: KEY=VALUE)
#[arg(short, long, value_name = "KEY=VALUE")]
pub env: Vec<String>,
/// Pod name
#[arg(short, long)]
pub name: Option<String>,
/// Volume size in GB
#[arg(long)]
pub volume_size: Option<u32>,
/// Dry run mode (validate request without deploying)
#[arg(long)]
pub dry_run: bool,
/// Skip confirmation prompt (auto-approve deployment)
#[arg(short = 'y', long)]
pub yes: bool,
}
/// Execute deploy command
pub(crate) async fn execute(config: &FoxhuntConfig, args: &DeployArgs) -> Result<()> {
utils::info("Starting RunPod deployment...");
// Create RunPod client
let client = RunPodClient::new(config.runpod.api_key.clone())?;
// List available GPUs
let gpus = client.list_gpus().await?;
info!("Found {} available GPU types", gpus.len());
// Select GPU
let gpu = select_best_gpu(&gpus, args.gpu_type.as_deref())?;
utils::success(&format!(
"Selected GPU: {} ({} GB VRAM, ${:.2}/hr)",
gpu.display_name.bold(),
gpu.memory_in_gb.unwrap_or(0),
gpu.secure_price.unwrap_or(0.0)
));
// Build deployment request
let request = build_deployment_request(args, config, &gpu)?;
// Validate deployment request
validate_deployment(&request)?;
// Display deployment summary
display_deployment_summary(&request, &gpu, args.dry_run);
// Dry run mode - just validate and exit
if args.dry_run {
utils::success("Dry run completed successfully. Request is valid.");
return Ok(());
}
// Confirm deployment (skip if --yes flag is set)
if !args.yes && !confirm_deployment(&request, &gpu)? {
utils::info("Deployment cancelled by user.");
return Ok(());
}
// Deploy pod
utils::info("Deploying pod to RunPod...");
let response = client.deploy_pod(request).await?;
// Display success message
display_deployment_result(&response, &gpu);
// Save pod ID for later reference
save_pod_id(&response.id)?;
Ok(())
}
/// Display deployment summary
#[allow(clippy::non_ascii_literal)]
fn display_deployment_summary(
request: &crate::runpod::PodDeploymentRequest,
gpu: &crate::runpod::GpuType,
dry_run: bool,
) {
let na_fallback = "N/A".to_owned();
println!();
utils::info(&format!(
"{} Deployment Summary",
if dry_run { "[DRY RUN]" } else { "" }
));
println!("{}", "─".repeat(60));
println!(" Pod Name: {}", request.name.bold());
println!(" GPU: {}", gpu.display_name.bold());
println!(
" VRAM: {} GB",
gpu.memory_in_gb.unwrap_or(0).to_string().bold()
);
println!(
" Cost: ${:.2}/hr",
gpu.secure_price.unwrap_or(0.0)
);
println!(
" Datacenter: {}",
request
.data_center_ids
.as_ref()
.and_then(|d| d.first())
.unwrap_or(&na_fallback)
.bold()
);
println!(" Image: {}", request.image_name.bold());
println!(
" Command: {}",
request
.docker_start_cmd
.as_ref()
.map(|cmd| cmd.join(" "))
.unwrap_or_default()
.bold()
);
println!(
" Container Disk: {} GB",
request.container_disk_in_gb.to_string().bold()
);
if let Some(env) = &request.env {
if !env.is_empty() {
println!(" Environment:");
for (key, value) in env {
println!(" {}={}", key, value);
}
}
}
println!("{}", "─".repeat(60));
println!();
}
/// Confirm deployment with user
fn confirm_deployment(
_request: &crate::runpod::PodDeploymentRequest,
gpu: &crate::runpod::GpuType,
) -> Result<bool> {
let cost_per_hr = gpu.secure_price.unwrap_or(0.0);
let estimated_daily = cost_per_hr * 24.0;
println!(
"{}",
format!(
"Estimated cost: ${:.2}/hr (${:.2}/day)",
cost_per_hr, estimated_daily
)
.yellow()
);
print!("Deploy pod? [y/N]: ");
std::io::Write::flush(&mut std::io::stdout())?;
let mut input = String::new();
std::io::stdin().read_line(&mut input)?;
Ok(input.trim().eq_ignore_ascii_case("y"))
}
/// Display deployment result
#[allow(clippy::non_ascii_literal)]
fn display_deployment_result(
response: &crate::runpod::PodDeploymentResponse,
gpu: &crate::runpod::GpuType,
) {
println!();
utils::success("Pod deployed successfully!");
println!();
println!("{}", "─".repeat(60));
println!(" Pod ID: {}", response.id.green().bold());
if let Some(machine) = &response.machine {
if let Some(gpu_type) = &machine.gpu_type_id {
println!(" GPU: {}", gpu_type.bold());
}
if let Some(vram) = machine.vram_gb {
println!(" VRAM: {} GB", vram.to_string().bold());
}
}
if let Some(runtime) = &response.runtime {
if let Some(datacenter) = &runtime.datacenter_id {
println!(" Datacenter: {}", datacenter.bold());
}
}
println!(
" Cost: ${:.2}/hr",
response.cost_per_hr.unwrap_or(gpu.secure_price.unwrap_or(0.0))
);
println!("{}", "─".repeat(60));
println!();
utils::info("Monitor your pod:");
println!(
" {}",
format!("foxhunt-deploy monitor {}", response.id).cyan()
);
println!();
utils::info("Terminate when done:");
println!(
" {}",
format!("foxhunt-deploy run terminate --pod-id {}", response.id).cyan()
);
println!();
}
/// Save pod ID to a file for later reference
fn save_pod_id(pod_id: &str) -> Result<()> {
use std::io::Write;
let pod_file = std::path::PathBuf::from(".last_pod_id");
let mut file = std::fs::File::create(&pod_file)?;
writeln!(file, "{}", pod_id)?;
info!("Saved pod ID to {}", pod_file.display());
Ok(())
}

48
foxhunt-deploy/src/cli/mod.rs
View File

@@ -1,48 +0,0 @@
pub(crate) mod build;
pub(crate) mod deploy;
pub(crate) mod monitor;
pub(crate) mod run;
use clap::{Parser, Subcommand};
use std::path::PathBuf;
/// Foxhunt RunPod deployment CLI
#[derive(Parser, Debug)]
#[command(
name = "foxhunt-deploy",
version,
about = "Foxhunt HFT Trading System - RunPod GPU Deployment CLI",
long_about = "Deploy and manage Foxhunt ML training workloads on RunPod GPU infrastructure.\n\
Supports Docker image building, pod deployment, log monitoring, and unified workflows."
)]
pub(crate) struct Cli {
/// Enable verbose logging
#[arg(short, long, global = true)]
pub verbose: bool,
/// Path to config file (default: ~/.runpod/config.toml)
#[arg(short, long, global = true, value_name = "FILE")]
pub config: Option<PathBuf>,
#[command(subcommand)]
pub command: Commands,
}
/// Available CLI commands
#[derive(Subcommand, Debug)]
pub(crate) enum Commands {
/// Initialize configuration file
Init,
/// Build Docker image
Build(build::BuildArgs),
/// Deploy pod to RunPod
Deploy(deploy::DeployArgs),
/// Monitor pod logs
Monitor(monitor::MonitorArgs),
/// Build and deploy in one command
Run(run::RunArgs),
}

56
foxhunt-deploy/src/cli/monitor.rs
View File

@@ -1,56 +0,0 @@
use crate::config::types::FoxhuntConfig;
use crate::error::Result;
use crate::s3::monitor::LogMonitor;
use crate::s3::S3LogClient;
use clap::Args;
/// Monitor pod logs from S3
#[derive(Args, Debug)]
pub(crate) struct MonitorArgs {
/// Pod ID to monitor
#[arg(required = true)]
pub pod_id: String,
/// Follow logs in real-time
#[arg(short, long)]
pub follow: bool,
/// Number of recent lines to show
#[arg(short, long)]
pub tail: Option<usize>,
/// Filter logs by pattern (regex)
#[arg(long)]
pub filter: Option<String>,
/// List available log files without displaying content
#[arg(short, long)]
pub list: bool,
}
/// Execute monitor command
pub(crate) async fn execute(config: &FoxhuntConfig, args: &MonitorArgs) -> Result<()> {
// Create S3 client
let s3_client = S3LogClient::new(&config.s3).await?;
// Create monitor
let monitor = LogMonitor::new(
s3_client,
args.pod_id.clone(),
config.s3.poll_interval_secs,
);
// Handle list mode
if args.list {
return monitor.list_logs().await;
}
// Stream logs
if args.follow {
monitor.tail_logs(args.tail, args.filter.clone()).await?;
} else {
monitor.show_recent_logs(args.tail).await?;
}
Ok(())
}

34
foxhunt-deploy/src/cli/run.rs
View File

@@ -1,34 +0,0 @@
use crate::config::types::FoxhuntConfig;
use crate::error::Result;
use clap::Args;
/// Build, deploy, and monitor in one command
#[derive(Args, Debug)]
pub(crate) struct RunArgs {
/// Training command to run
#[arg(short, long, required = true)]
pub command: String,
/// GPU type (e.g., "RTX A4000", "RTX 4090")
#[arg(short, long)]
pub gpu_type: Option<String>,
/// Docker image tag
#[arg(short, long, default_value = "latest")]
pub tag: String,
/// Skip Docker build step
#[arg(long)]
pub skip_build: bool,
/// Don't monitor logs after deployment
#[arg(long)]
pub no_monitor: bool,
}
/// Execute run command
pub(crate) async fn execute(_config: &FoxhuntConfig, _args: &RunArgs) -> Result<()> {
println!("Run command (Milestone 5)");
println!("This will build, deploy, and monitor in one unified workflow");
Ok(())
}

207
foxhunt-deploy/src/config/mod.rs
View File

@@ -1,207 +0,0 @@
pub(crate) mod types;
use crate::error::{FoxhuntError, Result};
use std::fs;
use std::path::PathBuf;
use types::FoxhuntConfig;
pub(crate) struct ConfigManager {
config_path: PathBuf,
}
impl ConfigManager {
/// Create a new ConfigManager with default config path
pub(crate) fn new() -> Result<Self> {
let config_path = Self::default_config_path()?;
Ok(Self { config_path })
}
/// Create a ConfigManager with custom config path
pub(crate) fn with_path(path: PathBuf) -> Self {
Self { config_path: path }
}
/// Get default config path: ~/.runpod/config.toml
pub(crate) fn default_config_path() -> Result<PathBuf> {
let home = home::home_dir().ok_or_else(|| {
FoxhuntError::Config("Could not determine home directory".to_owned())
})?;
Ok(home.join(".runpod").join("config.toml"))
}
/// Load configuration from file and override with environment variables
pub(crate) fn load(&self) -> Result<FoxhuntConfig> {
// Load from TOML file
let config = self.load_from_file()?;
// Override with environment variables from .env.runpod
self.override_with_env(config)
}
/// Load configuration from TOML file
fn load_from_file(&self) -> Result<FoxhuntConfig> {
if !self.config_path.exists() {
return Err(FoxhuntError::ConfigNotFound {
path: self.config_path.clone(),
});
}
let contents = fs::read_to_string(&self.config_path)?;
let config: FoxhuntConfig = toml::from_str(&contents)?;
Ok(config)
}
/// Override configuration with environment variables
fn override_with_env(&self, mut config: FoxhuntConfig) -> Result<FoxhuntConfig> {
// Try to load .env.runpod if it exists
drop(dotenvy::from_filename(".env.runpod"));
// Override RunPod config
if let Ok(api_key) = std::env::var("RUNPOD_API_KEY") {
config.runpod.api_key = api_key;
}
if let Ok(gpu_type) = std::env::var("RUNPOD_GPU_TYPE") {
config.runpod.default_gpu_type = gpu_type;
}
if let Ok(datacenter) = std::env::var("RUNPOD_DATACENTER") {
config.runpod.default_datacenter = datacenter;
}
// Override Docker config
if let Ok(registry) = std::env::var("DOCKER_REGISTRY") {
config.docker.registry = registry;
}
if let Ok(image_name) = std::env::var("DOCKER_IMAGE_NAME") {
config.docker.image_name = image_name;
}
if let Ok(tag) = std::env::var("DOCKER_TAG") {
config.docker.tag = tag;
}
// Override S3 config
if let Ok(endpoint) = std::env::var("S3_ENDPOINT") {
config.s3.endpoint = endpoint;
}
if let Ok(bucket) = std::env::var("S3_BUCKET") {
config.s3.bucket = bucket;
}
if let Ok(region) = std::env::var("S3_REGION") {
config.s3.region = region;
}
if let Ok(poll_interval) = std::env::var("S3_POLL_INTERVAL_SECS") {
if let Ok(interval) = poll_interval.parse::<u64>() {
config.s3.poll_interval_secs = interval;
}
}
// Override deployment defaults
if let Ok(disk) = std::env::var("CONTAINER_DISK_GB") {
if let Ok(disk_gb) = disk.parse::<u32>() {
config.defaults.container_disk_gb = disk_gb;
}
}
if let Ok(volume_path) = std::env::var("VOLUME_PATH") {
config.defaults.volume_path = volume_path;
}
if let Ok(volume_size) = std::env::var("VOLUME_SIZE_GB") {
if let Ok(size_gb) = volume_size.parse::<u32>() {
config.defaults.volume_size_gb = size_gb;
}
}
if let Ok(public_ip) = std::env::var("SUPPORT_PUBLIC_IP") {
if let Ok(support) = public_ip.parse::<bool>() {
config.defaults.support_public_ip = support;
}
}
Ok(config)
}
/// Validate required configuration fields
pub(crate) fn validate(config: &FoxhuntConfig) -> Result<()> {
// Validate API key
if config.runpod.api_key.is_empty()
|| config.runpod.api_key == "YOUR_RUNPOD_API_KEY_HERE"
{
return Err(FoxhuntError::MissingConfigField {
field: "runpod.api_key".to_owned(),
});
}
// Validate S3 bucket
if config.s3.bucket.is_empty() {
return Err(FoxhuntError::MissingConfigField {
field: "s3.bucket".to_owned(),
});
}
Ok(())
}
/// Create sample config file
pub(crate) fn create_sample_config() -> Result<PathBuf> {
let config_path = Self::default_config_path()?;
let config_dir = config_path
.parent()
.ok_or_else(|| FoxhuntError::Config("Invalid config path".to_owned()))?;
// Create config directory if it doesn't exist
fs::create_dir_all(config_dir)?;
// Create default config
let config = FoxhuntConfig::default();
let toml_str = toml::to_string_pretty(&config).map_err(|e| {
FoxhuntError::Config(format!("Failed to serialize config: {}", e))
})?;
// Write to file
fs::write(&config_path, toml_str)?;
Ok(config_path)
}
}
impl Default for ConfigManager {
fn default() -> Self {
Self::new().unwrap_or_else(|_| {
// Fallback to a reasonable default path if home dir detection fails
Self::with_path(PathBuf::from(".runpod/config.toml"))
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::env;
#[test]
fn test_default_config() {
let config = FoxhuntConfig::default();
assert_eq!(config.runpod.default_gpu_type, "RTX A4000");
assert_eq!(config.docker.registry, "jgrusewski");
assert_eq!(config.s3.bucket, "se3zdnb5o4");
}
#[test]
fn test_env_override() {
env::set_var("RUNPOD_API_KEY", "test_key");
let manager = ConfigManager::with_path(PathBuf::from("test_config.toml"));
let config = FoxhuntConfig::default();
let overridden = manager.override_with_env(config).unwrap();
assert_eq!(overridden.runpod.api_key, "test_key");
env::remove_var("RUNPOD_API_KEY");
}
#[test]
fn test_validation() {
let mut config = FoxhuntConfig::default();
// Should fail with default API key
assert!(ConfigManager::validate(&config).is_err());
// Should succeed with real API key
config.runpod.api_key = "real_api_key".to_owned();
assert!(ConfigManager::validate(&config).is_ok());
}
}

167
foxhunt-deploy/src/config/types.rs
View File

@@ -1,167 +0,0 @@
use serde::{Deserialize, Serialize};
/// Main configuration struct
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct FoxhuntConfig {
pub runpod: RunPodConfig,
pub docker: DockerConfig,
pub s3: S3Config,
pub defaults: DeploymentDefaults,
}
impl Default for FoxhuntConfig {
fn default() -> Self {
Self {
runpod: RunPodConfig::default(),
docker: DockerConfig::default(),
s3: S3Config::default(),
defaults: DeploymentDefaults::default(),
}
}
}
/// RunPod API configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct RunPodConfig {
#[serde(default = "default_api_key")]
pub api_key: String,
#[serde(default = "default_gpu_type")]
pub default_gpu_type: String,
#[serde(default = "default_datacenter")]
pub default_datacenter: String,
}
impl Default for RunPodConfig {
fn default() -> Self {
Self {
api_key: default_api_key(),
default_gpu_type: default_gpu_type(),
default_datacenter: default_datacenter(),
}
}
}
fn default_api_key() -> String {
"YOUR_RUNPOD_API_KEY_HERE".to_owned()
}
fn default_gpu_type() -> String {
"RTX A4000".to_owned()
}
fn default_datacenter() -> String {
"EUR-IS-1".to_owned()
}
/// Docker configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct DockerConfig {
#[serde(default = "default_registry")]
pub registry: String,
#[serde(default = "default_image_name")]
pub image_name: String,
#[serde(default = "default_tag")]
pub tag: String,
}
impl Default for DockerConfig {
fn default() -> Self {
Self {
registry: default_registry(),
image_name: default_image_name(),
tag: default_tag(),
}
}
}
fn default_registry() -> String {
"jgrusewski".to_owned()
}
fn default_image_name() -> String {
"foxhunt".to_owned()
}
fn default_tag() -> String {
"latest".to_owned()
}
/// S3 configuration for log monitoring
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct S3Config {
#[serde(default = "default_endpoint")]
pub endpoint: String,
#[serde(default = "default_bucket")]
pub bucket: String,
#[serde(default = "default_region")]
pub region: String,
#[serde(default = "default_poll_interval")]
pub poll_interval_secs: u64,
}
impl Default for S3Config {
fn default() -> Self {
Self {
endpoint: default_endpoint(),
bucket: default_bucket(),
region: default_region(),
poll_interval_secs: default_poll_interval(),
}
}
}
fn default_endpoint() -> String {
"https://s3api-eur-is-1.runpod.io".to_owned()
}
fn default_bucket() -> String {
"se3zdnb5o4".to_owned()
}
fn default_region() -> String {
"us-east-1".to_owned()
}
fn default_poll_interval() -> u64 {
5
}
/// Deployment defaults
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct DeploymentDefaults {
#[serde(default = "default_container_disk_gb")]
pub container_disk_gb: u32,
#[serde(default = "default_volume_path")]
pub volume_path: String,
#[serde(default = "default_volume_size_gb")]
pub volume_size_gb: u32,
#[serde(default = "default_support_public_ip")]
pub support_public_ip: bool,
}
impl Default for DeploymentDefaults {
fn default() -> Self {
Self {
container_disk_gb: default_container_disk_gb(),
volume_path: default_volume_path(),
volume_size_gb: default_volume_size_gb(),
support_public_ip: default_support_public_ip(),
}
}
}
fn default_container_disk_gb() -> u32 {
50
}
fn default_volume_path() -> String {
"/runpod-volume".to_owned()
}
fn default_volume_size_gb() -> u32 {
100
}
fn default_support_public_ip() -> bool {
false
}

226
foxhunt-deploy/src/docker/build.rs
View File

@@ -1,226 +0,0 @@
use crate::error::{FoxhuntError, Result};
use crate::utils;
use indicatif::{ProgressBar, ProgressStyle};
use std::io::{BufRead, BufReader};
use std::process::{Command, Stdio};
use std::time::Duration;
/// Options for building a Docker image
#[derive(Debug, Clone)]
pub(crate) struct DockerBuildOptions {
/// Docker image tag (e.g., "jgrusewski/foxhunt:latest")
pub tag: String,
/// Path to Dockerfile
pub dockerfile: String,
/// Build context directory
pub context: String,
/// Disable build cache
pub no_cache: bool,
/// Additional build arguments
pub build_args: Vec<(String, String)>,
}
impl DockerBuildOptions {
pub(crate) fn new(tag: impl Into<String>) -> Self {
Self {
tag: tag.into(),
dockerfile: "Dockerfile.foxhunt-build".to_owned(),
context: ".".to_owned(),
no_cache: false,
build_args: Vec::new(),
}
}
pub(crate) fn dockerfile(mut self, path: impl Into<String>) -> Self {
self.dockerfile = path.into();
self
}
pub(crate) fn context(mut self, path: impl Into<String>) -> Self {
self.context = path.into();
self
}
pub(crate) fn no_cache(mut self, no_cache: bool) -> Self {
self.no_cache = no_cache;
self
}
#[allow(dead_code)]
pub(crate) fn build_arg(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
self.build_args.push((key.into(), value.into()));
self
}
}
/// Build a Docker image and return the image ID
#[allow(clippy::non_ascii_literal)]
pub(crate) fn build_image(options: &DockerBuildOptions) -> Result<String> {
// Verify Dockerfile exists
let dockerfile_path = std::path::Path::new(&options.dockerfile);
if !dockerfile_path.exists() {
return Err(FoxhuntError::Docker(format!(
"Dockerfile not found: {}",
options.dockerfile
)));
}
// Verify build context exists
let context_path = std::path::Path::new(&options.context);
if !context_path.exists() {
return Err(FoxhuntError::Docker(format!(
"Build context directory not found: {}",
options.context
)));
}
utils::info(&format!("Building Docker image: {}", options.tag));
utils::info(&format!("Dockerfile: {}", options.dockerfile));
utils::info(&format!("Context: {}", options.context));
// Create progress bar
let pb = ProgressBar::new_spinner();
pb.set_style(
ProgressStyle::default_spinner()
.template("{spinner:.blue} {msg}")
.unwrap()
.tick_strings(&["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]),
);
pb.enable_steady_tick(Duration::from_millis(100));
pb.set_message("Starting Docker build...");
// Build docker command
let mut cmd = Command::new("docker");
cmd.arg("build");
cmd.arg("--tag").arg(&options.tag);
cmd.arg("--file").arg(&options.dockerfile);
if options.no_cache {
cmd.arg("--no-cache");
}
// Add build args
for (key, value) in &options.build_args {
cmd.arg("--build-arg").arg(format!("{}={}", key, value));
}
// Add context last
cmd.arg(&options.context);
// Configure to capture output
cmd.stdout(Stdio::piped());
cmd.stderr(Stdio::piped());
// Spawn the process
let mut child = cmd
.spawn()
.map_err(|e| FoxhuntError::Docker(format!("Failed to spawn docker build: {}", e)))?;
// Stream stdout
if let Some(stdout) = child.stdout.take() {
let reader = BufReader::new(stdout);
for raw_line in reader.lines() {
if let Ok(output_line) = raw_line {
// Update progress bar with build step
if output_line.contains("Step ") {
pb.set_message(output_line.clone());
}
tracing::debug!("docker build: {}", output_line);
}
}
}
// Wait for completion
let status = child
.wait()
.map_err(|e| FoxhuntError::Docker(format!("Failed to wait for docker build: {}", e)))?;
pb.finish_and_clear();
if !status.success() {
// Try to get stderr for error details
let stderr = if let Some(stderr) = child.stderr {
let reader = BufReader::new(stderr);
let mut error_msg = String::new();
for err_line in reader.lines().flatten() {
error_msg.push_str(&err_line);
error_msg.push('\n');
}
error_msg
} else {
"Unknown error".to_owned()
};
return Err(FoxhuntError::Docker(format!(
"Docker build failed with exit code {:?}:\n{}",
status.code(),
stderr
)));
}
// Get the image ID
let image_id = get_image_id(&options.tag)?;
utils::success(&format!("Built image: {} ({})", options.tag, image_id));
Ok(image_id)
}
/// Get the image ID for a given tag
fn get_image_id(tag: &str) -> Result<String> {
let output = Command::new("docker")
.args(["images", "-q", tag])
.output()
.map_err(|e| FoxhuntError::Docker(format!("Failed to get image ID: {}", e)))?;
if !output.status.success() {
return Err(FoxhuntError::Docker(
"Failed to retrieve image ID after build".to_owned(),
));
}
let image_id = String::from_utf8_lossy(&output.stdout)
.trim()
.to_owned();
if image_id.is_empty() {
return Err(FoxhuntError::Docker(
"Image ID is empty after build".to_owned(),
));
}
Ok(image_id)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_build_options_builder() {
let options = DockerBuildOptions::new("test:latest")
.dockerfile("Dockerfile.test")
.context("/tmp/test")
.no_cache(true)
.build_arg("VERSION", "1.0.0");
assert_eq!(options.tag, "test:latest");
assert_eq!(options.dockerfile, "Dockerfile.test");
assert_eq!(options.context, "/tmp/test");
assert!(options.no_cache);
assert_eq!(options.build_args.len(), 1);
assert_eq!(options.build_args[0].0, "VERSION");
assert_eq!(options.build_args[0].1, "1.0.0");
}
#[test]
fn test_build_options_defaults() {
let options = DockerBuildOptions::new("test:latest");
assert_eq!(options.tag, "test:latest");
assert_eq!(options.dockerfile, "Dockerfile.foxhunt-build");
assert_eq!(options.context, ".");
assert!(!options.no_cache);
assert!(options.build_args.is_empty());
}
}

65
foxhunt-deploy/src/docker/mod.rs
View File

@@ -1,65 +0,0 @@
pub(crate) mod build;
pub(crate) mod push;
use crate::error::{FoxhuntError, Result};
use std::process::Command;
/// Verify Docker is installed and daemon is running
pub(crate) fn verify_docker_available() -> Result<()> {
// Check if docker command exists
let which_result = Command::new("which")
.arg("docker")
.output()
.map_err(|e| FoxhuntError::Docker(format!("Failed to check for Docker installation: {}", e)))?;
if !which_result.status.success() {
return Err(FoxhuntError::Docker(
"Docker is not installed. Please install Docker Desktop or Docker Engine.".to_owned(),
));
}
// Check if Docker daemon is running
let version_result = Command::new("docker")
.arg("version")
.arg("--format")
.arg("{{.Server.Version}}")
.output()
.map_err(|e| FoxhuntError::Docker(format!("Failed to check Docker daemon: {}", e)))?;
if !version_result.status.success() {
return Err(FoxhuntError::Docker(
"Docker daemon is not running. Please start Docker Desktop or Docker service.".to_owned(),
));
}
Ok(())
}
/// Check if a Docker image exists locally
pub(crate) fn image_exists(tag: &str) -> Result<bool> {
let output = Command::new("docker")
.args(["images", "-q", tag])
.output()
.map_err(|e| FoxhuntError::Docker(format!("Failed to check if image exists: {}", e)))?;
Ok(!output.stdout.is_empty())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_verify_docker_available() {
// This will succeed or fail depending on Docker installation
// We don't assert here since it's environment-dependent
let _ = verify_docker_available();
}
#[test]
fn test_image_exists() {
// Test with a non-existent image
let result = image_exists("nonexistent-image-12345:latest");
assert!(result.is_ok());
}
}

130
foxhunt-deploy/src/docker/push.rs
View File

@@ -1,130 +0,0 @@
use crate::error::{FoxhuntError, Result};
use crate::utils;
use indicatif::{ProgressBar, ProgressStyle};
use std::io::{BufRead, BufReader};
use std::process::{Command, Stdio};
use std::time::Duration;
/// Push a Docker image to a registry
#[allow(clippy::non_ascii_literal)]
pub(crate) fn push_image(tag: &str) -> Result<()> {
// Verify image exists locally
if !super::image_exists(tag)? {
return Err(FoxhuntError::Docker(format!(
"Image '{}' does not exist locally. Build it first.",
tag
)));
}
utils::info(&format!("Pushing Docker image: {}", tag));
// Create progress bar
let pb = ProgressBar::new_spinner();
pb.set_style(
ProgressStyle::default_spinner()
.template("{spinner:.blue} {msg}")
.unwrap()
.tick_strings(&["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]),
);
pb.enable_steady_tick(Duration::from_millis(100));
pb.set_message("Pushing to registry...");
// Build docker push command
let mut cmd = Command::new("docker");
cmd.arg("push");
cmd.arg(tag);
// Configure to capture output
cmd.stdout(Stdio::piped());
cmd.stderr(Stdio::piped());
// Spawn the process
let mut child = cmd
.spawn()
.map_err(|e| FoxhuntError::Docker(format!("Failed to spawn docker push: {}", e)))?;
// Stream stdout to show progress
if let Some(stdout) = child.stdout.take() {
let reader = BufReader::new(stdout);
for raw_line in reader.lines() {
if let Ok(output_line) = raw_line {
// Update progress bar with push progress
if output_line.contains("Pushing") || output_line.contains("Pushed") || output_line.contains("Waiting") {
pb.set_message(output_line.clone());
}
tracing::debug!("docker push: {}", output_line);
}
}
}
// Wait for completion
let status = child
.wait()
.map_err(|e| FoxhuntError::Docker(format!("Failed to wait for docker push: {}", e)))?;
pb.finish_and_clear();
if !status.success() {
// Try to get stderr for error details
let stderr = if let Some(stderr) = child.stderr {
let reader = BufReader::new(stderr);
let mut error_msg = String::new();
for err_line in reader.lines().flatten() {
error_msg.push_str(&err_line);
error_msg.push('\n');
}
error_msg
} else {
"Unknown error".to_owned()
};
// Check for authentication errors
if stderr.contains("authentication") || stderr.contains("unauthorized") {
return Err(FoxhuntError::Docker(format!(
"Docker registry authentication failed. Please run 'docker login' first.\n{}",
stderr
)));
}
return Err(FoxhuntError::Docker(format!(
"Docker push failed with exit code {:?}:\n{}",
status.code(),
stderr
)));
}
utils::success(&format!("Successfully pushed: {}", tag));
Ok(())
}
/// Check if user is logged in to Docker registry
#[allow(dead_code)]
pub(crate) fn check_registry_auth(registry: &str) -> Result<bool> {
// Try to get credentials from Docker config
let output = Command::new("docker")
.args(["login", "--help"])
.output()
.map_err(|e| FoxhuntError::Docker(format!("Failed to check Docker login status: {}", e)))?;
if !output.status.success() {
return Ok(false);
}
// For simplicity, we assume if docker login command exists, auth is possible
// A more robust check would parse ~/.docker/config.json
tracing::debug!("Registry auth check for: {}", registry);
Ok(true)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_check_registry_auth() {
// This will succeed or fail depending on Docker installation
let result = check_registry_auth("docker.io");
assert!(result.is_ok());
}
}

45
foxhunt-deploy/src/error.rs
View File

@@ -1,45 +0,0 @@
use std::path::PathBuf;
use thiserror::Error;
/// Custom Result type for foxhunt-deploy
pub(crate) type Result<T> = std::result::Result<T, FoxhuntError>;
/// Unified error type for all operations
#[derive(Error, Debug)]
pub(crate) enum FoxhuntError {
#[error("Configuration error: {0}")]
Config(String),
#[error("Config file not found at {path:?}. Run 'foxhunt-deploy init' to create one.")]
ConfigNotFound { path: PathBuf },
#[error("Missing required config field: {field}")]
MissingConfigField { field: String },
#[error("Docker command failed: {0}")]
Docker(String),
#[error("RunPod API error: {status} - {message}")]
RunPodApi { status: u16, message: String },
#[error("S3 error: {0}")]
S3(String),
#[error("IO error: {0}")]
Io(#[from] std::io::Error),
#[error("HTTP request failed: {0}")]
Http(#[from] reqwest::Error),
#[error("Serialization error: {0}")]
Serde(#[from] serde_json::Error),
#[error("TOML parsing error: {0}")]
Toml(#[from] toml::de::Error),
#[error("Invalid argument: {0}")]
InvalidArgument(String),
#[error("Unknown error: {0}")]
Unknown(String),
}

102
foxhunt-deploy/src/main.rs
View File

@@ -1,102 +0,0 @@
mod cli;
mod config;
mod docker;
mod error;
mod runpod;
mod s3;
mod utils;
use clap::Parser;
use cli::{Cli, Commands};
use config::ConfigManager;
use error::Result;
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
#[tokio::main]
async fn main() {
if let Err(e) = run().await {
utils::error(format!("Error: {}", e));
std::process::exit(1);
}
}
async fn run() -> Result<()> {
// Parse CLI arguments
let cli = Cli::parse();
// Initialize logging
init_logging(cli.verbose);
// Handle init command separately (doesn't need config)
if matches!(cli.command, Commands::Init) {
return handle_init();
}
// Load configuration
let config_manager = if let Some(config_path) = cli.config {
ConfigManager::with_path(config_path)
} else {
ConfigManager::new()?
};
let config = config_manager.load()?;
// Validate configuration
ConfigManager::validate(&config)?;
// Route to appropriate subcommand
match cli.command {
Commands::Init => {
// Already handled above; return Ok to satisfy the match arm
Ok(())
}
Commands::Build(args) => {
cli::build::execute(&config, &args).await?;
Ok(())
}
Commands::Deploy(args) => {
cli::deploy::execute(&config, &args).await?;
Ok(())
}
Commands::Monitor(args) => {
cli::monitor::execute(&config, &args).await?;
Ok(())
}
Commands::Run(args) => {
cli::run::execute(&config, &args).await?;
Ok(())
}
}
}
/// Initialize logging based on verbosity
fn init_logging(verbose: bool) {
let filter = if verbose {
EnvFilter::try_from_default_env()
.unwrap_or_else(|_| EnvFilter::new("foxhunt_deploy=debug,info"))
} else {
EnvFilter::try_from_default_env()
.unwrap_or_else(|_| EnvFilter::new("foxhunt_deploy=info"))
};
tracing_subscriber::registry()
.with(filter)
.with(tracing_subscriber::fmt::layer().with_target(false))
.init();
}
/// Handle init command
fn handle_init() -> Result<()> {
utils::info("Initializing foxhunt-deploy configuration...");
let config_path = ConfigManager::create_sample_config()?;
utils::success(format!(
"Created sample configuration at: {}",
config_path.display()
));
utils::info("Please edit the config file and add your RunPod API key.");
utils::info("You can also override settings with a .env.runpod file or environment variables.");
Ok(())
}

120
foxhunt-deploy/src/runpod/client.rs
View File

@@ -1,120 +0,0 @@
use super::types::{
ApiError, GpuType, PodDeploymentRequest, PodDeploymentResponse,
};
use crate::error::{FoxhuntError, Result};
use reqwest::{header, Client};
use tracing::{debug, info};
const RUNPOD_REST_BASE: &str = "https://rest.runpod.io/v1";
/// RunPod API client
pub(crate) struct RunPodClient {
api_key: String,
http_client: Client,
}
impl RunPodClient {
/// Create a new RunPod client
pub(crate) fn new(api_key: String) -> Result<Self> {
if api_key.is_empty() || api_key == "YOUR_RUNPOD_API_KEY_HERE" {
return Err(FoxhuntError::Config(
"RunPod API key not set. Please configure it in foxhunt.toml or .env.runpod"
.to_owned(),
));
}
let http_client = Client::builder()
.timeout(std::time::Duration::from_secs(30))
.build()?;
Ok(Self {
api_key,
http_client,
})
}
/// List available GPU types (simplified implementation)
/// In production, this would use the GraphQL API to query gpuTypes
pub(crate) async fn list_gpus(&self) -> Result<Vec<GpuType>> {
info!("Fetching available GPU types from RunPod...");
// For now, return hardcoded list of common GPUs
// TODO: Implement GraphQL query to fetch real-time GPU availability
Ok(vec![
GpuType {
id: "NVIDIA RTX A4000".to_owned(),
display_name: "RTX A4000".to_owned(),
memory_in_gb: Some(16),
secure_price: Some(0.25),
community_price: Some(0.18),
lowest_price: None,
},
GpuType {
id: "NVIDIA RTX 4090".to_owned(),
display_name: "RTX 4090".to_owned(),
memory_in_gb: Some(24),
secure_price: Some(0.59),
community_price: Some(0.44),
lowest_price: None,
},
GpuType {
id: "NVIDIA A40".to_owned(),
display_name: "A40".to_owned(),
memory_in_gb: Some(48),
secure_price: Some(0.45),
community_price: Some(0.35),
lowest_price: None,
},
])
}
/// Deploy a new pod
pub(crate) async fn deploy_pod(&self, request: PodDeploymentRequest) -> Result<PodDeploymentResponse> {
info!("Deploying pod: {}", request.name);
debug!("Deployment request: {:?}", request);
let url = format!("{}/pods", RUNPOD_REST_BASE);
let response = self
.http_client
.post(&url)
.header(header::AUTHORIZATION, format!("Bearer {}", self.api_key))
.header(header::CONTENT_TYPE, "application/json")
.json(&request)
.send()
.await?;
let status = response.status();
let body = response.text().await?;
debug!("API response (status {}): {}", status, body);
if status.is_success() {
let pod_response: PodDeploymentResponse = serde_json::from_str(&body)
.map_err(|e| {
FoxhuntError::Unknown(format!(
"Failed to parse deployment response: {}. Body: {}",
e, body
))
})?;
info!("Pod deployed successfully: {}", pod_response.id);
Ok(pod_response)
} else {
// Try to parse error response
let error_msg = if let Ok(api_error) = serde_json::from_str::<ApiError>(&body) {
api_error
.error
.or(api_error.message)
.unwrap_or_else(|| body.clone())
} else {
body
};
Err(FoxhuntError::RunPodApi {
status: status.as_u16(),
message: error_msg,
})
}
}
}

304
foxhunt-deploy/src/runpod/deployment.rs
View File

@@ -1,304 +0,0 @@
use super::types::{GpuType, PodDeploymentRequest};
use crate::cli::deploy::DeployArgs;
use crate::config::types::FoxhuntConfig;
use crate::error::{FoxhuntError, Result};
use std::collections::HashMap;
use tracing::{debug, info};
/// Build a deployment request from CLI args and config
pub(crate) fn build_deployment_request(
args: &DeployArgs,
config: &FoxhuntConfig,
gpu: &GpuType,
) -> Result<PodDeploymentRequest> {
// Parse command into docker start command
let docker_start_cmd = parse_command(&args.command)?;
// Build environment variables
let mut env = HashMap::new();
for env_var in &args.env {
let parts: Vec<&str> = env_var.splitn(2, '=').collect();
if parts.len() != 2 {
return Err(FoxhuntError::InvalidArgument(format!(
"Invalid environment variable format: {}. Expected KEY=VALUE",
env_var
)));
}
env.insert(parts[0].to_owned(), parts[1].to_owned());
}
// Determine datacenter
let datacenter = args
.datacenter
.clone()
.unwrap_or_else(|| config.runpod.default_datacenter.clone());
// Build image name
let image_name = format!(
"{}/{}:{}",
config.docker.registry, config.docker.image_name, args.tag
);
// Generate pod name
let pod_name = args.name.as_ref().cloned().unwrap_or_else(|| {
format!(
"foxhunt-{}",
chrono::Utc::now().format("%Y%m%d-%H%M%S")
)
});
// Get volume configuration (unused for now, but reserved for future use)
let _volume_size = args
.volume_size
.unwrap_or(config.defaults.volume_size_gb);
info!("Building deployment request:");
info!(" Name: {}", pod_name);
info!(" GPU: {}", gpu.display_name);
info!(" Datacenter: {}", datacenter);
info!(" Image: {}", image_name);
info!(" Command: {:?}", docker_start_cmd);
Ok(PodDeploymentRequest {
cloud_type: "SECURE".to_owned(),
compute_type: "GPU".to_owned(),
data_center_ids: Some(vec![datacenter]),
gpu_type_ids: vec![gpu.id.clone()],
gpu_count: 1,
name: pod_name,
image_name,
container_disk_in_gb: config.defaults.container_disk_gb,
network_volume_id: Some(config.s3.bucket.clone()),
volume_mount_path: Some(config.defaults.volume_path.clone()),
docker_start_cmd: Some(docker_start_cmd),
container_registry_auth_id: Some("cmh3ya1710001jo02vwqtisbf".to_owned()),
ports: Some(vec!["8888/http".to_owned(), "22/tcp".to_owned()]),
interruptible: Some(false),
env: if env.is_empty() { None } else { Some(env) },
support_public_ip: Some(config.defaults.support_public_ip),
})
}
/// Parse command string into docker start command array
fn parse_command(command: &str) -> Result<Vec<String>> {
// Simple split by whitespace (could be enhanced for quoted arguments)
let parts: Vec<String> = command
.split_whitespace()
.map(|s| s.to_owned())
.collect();
if parts.is_empty() {
return Err(FoxhuntError::InvalidArgument(
"Command cannot be empty".to_owned(),
));
}
debug!("Parsed command: {:?}", parts);
Ok(parts)
}
/// Select the best GPU based on availability and price
pub(crate) fn select_best_gpu(gpus: &[GpuType], preferred_type: Option<&str>) -> Result<GpuType> {
if gpus.is_empty() {
return Err(FoxhuntError::Unknown(
"No GPUs available from RunPod".to_owned(),
));
}
// If user specified a GPU type, find it
if let Some(preferred) = preferred_type {
if let Some(gpu) = gpus.iter().find(|g| {
g.id.to_lowercase().contains(&preferred.to_lowercase())
|| g.display_name
.to_lowercase()
.contains(&preferred.to_lowercase())
}) {
info!("Selected user-preferred GPU: {}", gpu.display_name);
return Ok(gpu.clone());
} else {
return Err(FoxhuntError::InvalidArgument(format!(
"GPU type '{}' not found. Available: {}",
preferred,
gpus.iter()
.map(|g| g.display_name.as_str())
.collect::<Vec<_>>()
.join(", ")
)));
}
}
// Auto-select: prefer RTX A4000 (good balance of price and performance)
if let Some(a4000) = gpus
.iter()
.find(|g| g.id.contains("A4000") || g.display_name.contains("A4000"))
{
info!(
"Auto-selected RTX A4000 (best value): ${:.2}/hr",
a4000.secure_price.unwrap_or(0.0)
);
return Ok(a4000.clone());
}
// Fallback to cheapest GPU
let cheapest = gpus
.iter()
.min_by(|a, b| {
let price_a = a.secure_price.unwrap_or(f64::MAX);
let price_b = b.secure_price.unwrap_or(f64::MAX);
price_a
.partial_cmp(&price_b)
.unwrap_or(std::cmp::Ordering::Equal)
})
.unwrap(); // Safe because we checked gpus.is_empty() above
info!(
"Auto-selected cheapest GPU: {} (${:.2}/hr)",
cheapest.display_name,
cheapest.secure_price.unwrap_or(0.0)
);
Ok(cheapest.clone())
}
/// Validate deployment request before submitting
pub(crate) fn validate_deployment(request: &PodDeploymentRequest) -> Result<()> {
// Validate GPU count
if request.gpu_count == 0 {
return Err(FoxhuntError::InvalidArgument(
"GPU count must be at least 1".to_owned(),
));
}
// Validate container disk size
if request.container_disk_in_gb < 10 {
return Err(FoxhuntError::InvalidArgument(
"Container disk size must be at least 10GB".to_owned(),
));
}
// Validate docker command
if let Some(cmd) = &request.docker_start_cmd {
if cmd.is_empty() {
return Err(FoxhuntError::InvalidArgument(
"Docker start command cannot be empty".to_owned(),
));
}
}
debug!("Deployment validation passed");
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_command() {
let result = parse_command("train_ppo --epochs 100").unwrap();
assert_eq!(result, vec!["train_ppo", "--epochs", "100"]);
}
#[test]
fn test_parse_command_empty() {
let result = parse_command("");
assert!(result.is_err());
}
#[test]
fn test_select_best_gpu_preferred() {
let gpus = vec![
GpuType {
id: "NVIDIA RTX A4000".to_owned(),
display_name: "RTX A4000".to_owned(),
memory_in_gb: Some(16),
secure_price: Some(0.25),
community_price: None,
lowest_price: None,
},
GpuType {
id: "NVIDIA RTX 4090".to_owned(),
display_name: "RTX 4090".to_owned(),
memory_in_gb: Some(24),
secure_price: Some(0.59),
community_price: None,
lowest_price: None,
},
];
let result = select_best_gpu(&gpus, Some("4090")).unwrap();
assert_eq!(result.display_name, "RTX 4090");
}
#[test]
fn test_select_best_gpu_auto() {
let gpus = vec![
GpuType {
id: "NVIDIA RTX A4000".to_owned(),
display_name: "RTX A4000".to_owned(),
memory_in_gb: Some(16),
secure_price: Some(0.25),
community_price: None,
lowest_price: None,
},
GpuType {
id: "NVIDIA RTX 4090".to_owned(),
display_name: "RTX 4090".to_owned(),
memory_in_gb: Some(24),
secure_price: Some(0.59),
community_price: None,
lowest_price: None,
},
];
let result = select_best_gpu(&gpus, None).unwrap();
assert_eq!(result.display_name, "RTX A4000"); // Should prefer A4000
}
#[test]
fn test_validate_deployment() {
let request = PodDeploymentRequest {
cloud_type: "SECURE".to_owned(),
compute_type: "GPU".to_owned(),
data_center_ids: Some(vec!["EUR-IS-1".to_owned()]),
gpu_type_ids: vec!["NVIDIA RTX A4000".to_owned()],
gpu_count: 1,
name: "test-pod".to_owned(),
image_name: "jgrusewski/foxhunt:latest".to_owned(),
container_disk_in_gb: 50,
network_volume_id: None,
volume_mount_path: None,
docker_start_cmd: Some(vec!["train_ppo".to_owned()]),
container_registry_auth_id: None,
ports: None,
interruptible: None,
env: None,
support_public_ip: None,
};
assert!(validate_deployment(&request).is_ok());
}
#[test]
fn test_validate_deployment_invalid_gpu_count() {
let request = PodDeploymentRequest {
cloud_type: "SECURE".to_owned(),
compute_type: "GPU".to_owned(),
data_center_ids: None,
gpu_type_ids: vec!["NVIDIA RTX A4000".to_owned()],
gpu_count: 0, // Invalid
name: "test-pod".to_owned(),
image_name: "jgrusewski/foxhunt:latest".to_owned(),
container_disk_in_gb: 50,
network_volume_id: None,
volume_mount_path: None,
docker_start_cmd: Some(vec!["train_ppo".to_owned()]),
container_registry_auth_id: None,
ports: None,
interruptible: None,
env: None,
support_public_ip: None,
};
assert!(validate_deployment(&request).is_err());
}
}

8
foxhunt-deploy/src/runpod/mod.rs
View File

@@ -1,8 +0,0 @@
pub(crate) mod client;
pub(crate) mod deployment;
pub(crate) mod types;
// Re-export main types for convenience
pub(crate) use client::RunPodClient;
pub(crate) use deployment::{build_deployment_request, select_best_gpu, validate_deployment};
pub(crate) use types::{GpuType, PodDeploymentRequest, PodDeploymentResponse};

168
foxhunt-deploy/src/runpod/types.rs
View File

@@ -1,168 +0,0 @@
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
/// Request payload for deploying a pod to RunPod
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct PodDeploymentRequest {
/// Cloud type (SECURE, COMMUNITY, ALL)
pub cloud_type: String,
/// Compute type (GPU, CPU)
pub compute_type: String,
/// Datacenter IDs to prefer
#[serde(skip_serializing_if = "Option::is_none")]
pub data_center_ids: Option<Vec<String>>,
/// GPU type IDs
pub gpu_type_ids: Vec<String>,
/// Number of GPUs
pub gpu_count: u32,
/// Pod name
pub name: String,
/// Docker image name
pub image_name: String,
/// Container disk size in GB
pub container_disk_in_gb: u32,
/// Network volume ID
#[serde(skip_serializing_if = "Option::is_none")]
pub network_volume_id: Option<String>,
/// Volume mount path
#[serde(skip_serializing_if = "Option::is_none")]
pub volume_mount_path: Option<String>,
/// Docker start command
#[serde(skip_serializing_if = "Option::is_none")]
pub docker_start_cmd: Option<Vec<String>>,
/// Container registry auth ID
#[serde(skip_serializing_if = "Option::is_none")]
pub container_registry_auth_id: Option<String>,
/// Exposed ports
#[serde(skip_serializing_if = "Option::is_none")]
pub ports: Option<Vec<String>>,
/// Allow interruptible instances
#[serde(skip_serializing_if = "Option::is_none")]
pub interruptible: Option<bool>,
/// Environment variables
#[serde(skip_serializing_if = "Option::is_none")]
pub env: Option<HashMap<String, String>>,
/// Support public IP
#[serde(skip_serializing_if = "Option::is_none")]
pub support_public_ip: Option<bool>,
}
/// Response from RunPod pod deployment
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct PodDeploymentResponse {
/// Pod ID
pub id: String,
/// Machine configuration
#[serde(skip_serializing_if = "Option::is_none")]
pub machine: Option<MachineInfo>,
/// Runtime information
#[serde(skip_serializing_if = "Option::is_none")]
pub runtime: Option<RuntimeInfo>,
/// Cost per hour
#[serde(skip_serializing_if = "Option::is_none")]
pub cost_per_hr: Option<f64>,
}
/// Machine information
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct MachineInfo {
/// GPU type
#[serde(skip_serializing_if = "Option::is_none")]
pub gpu_type_id: Option<String>,
/// GPU count
#[serde(skip_serializing_if = "Option::is_none")]
pub gpu_count: Option<u32>,
/// VRAM per GPU in GB
#[serde(skip_serializing_if = "Option::is_none")]
pub vram_gb: Option<u32>,
}
/// Runtime information
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct RuntimeInfo {
/// Datacenter ID
#[serde(skip_serializing_if = "Option::is_none")]
pub datacenter_id: Option<String>,
}
/// GPU type information
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct GpuType {
/// GPU type ID
pub id: String,
/// Display name
pub display_name: String,
/// VRAM in GB
#[serde(skip_serializing_if = "Option::is_none")]
pub memory_in_gb: Option<u32>,
/// Secure cloud price per hour
#[serde(skip_serializing_if = "Option::is_none")]
pub secure_price: Option<f64>,
/// Community cloud price per hour
#[serde(skip_serializing_if = "Option::is_none")]
pub community_price: Option<f64>,
/// Lowest price (used for auto-selection)
#[serde(skip_serializing_if = "Option::is_none")]
pub lowest_price: Option<LowestPrice>,
}
/// Lowest price information
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct LowestPrice {
/// Minimum price per hour
pub min_price_per_hr: f64,
/// Whether it's interruptible
#[serde(skip_serializing_if = "Option::is_none")]
pub interruptible: Option<bool>,
}
/// RunPod API error response
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct ApiError {
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub message: Option<String>,
}
/// Pod termination response
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct PodTerminationResponse {
pub id: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub status: Option<String>,
}

165
foxhunt-deploy/src/s3/mod.rs
View File

@@ -1,165 +0,0 @@
use crate::config::types::S3Config;
use crate::error::{FoxhuntError, Result};
use aws_config::meta::region::RegionProviderChain;
use aws_config::BehaviorVersion;
use aws_sdk_s3::config::Credentials;
use aws_sdk_s3::Client;
use std::env;
pub(crate) mod monitor;
pub(crate) mod parser;
/// S3 client wrapper for log operations
#[derive(Clone)]
pub(crate) struct S3LogClient {
client: Client,
bucket: String,
}
impl S3LogClient {
/// Create a new S3 client with RunPod configuration
pub(crate) async fn new(config: &S3Config) -> Result<Self> {
// Get AWS credentials from environment
let access_key = env::var("AWS_ACCESS_KEY_ID")
.map_err(|_| FoxhuntError::Config("AWS_ACCESS_KEY_ID not set".to_owned()))?;
let secret_key = env::var("AWS_SECRET_ACCESS_KEY")
.map_err(|_| FoxhuntError::Config("AWS_SECRET_ACCESS_KEY not set".to_owned()))?;
// Create credentials
let credentials = Credentials::new(access_key, secret_key, None, None, "runpod-s3");
// Configure region
let region_provider = RegionProviderChain::first_try(
aws_sdk_s3::config::Region::new(config.region.clone()),
);
// Build AWS config with custom endpoint
let sdk_config = aws_config::defaults(BehaviorVersion::latest())
.region(region_provider)
.credentials_provider(credentials)
.load()
.await;
// Build S3 client with custom endpoint
let s3_config = aws_sdk_s3::config::Builder::from(&sdk_config)
.endpoint_url(&config.endpoint)
.force_path_style(true)
.build();
let client = Client::from_conf(s3_config);
Ok(Self {
client,
bucket: config.bucket.clone(),
})
}
/// List all log files for a specific pod
pub(crate) async fn list_log_files(&self, pod_id: &str) -> Result<Vec<String>> {
let prefix = format!("ml_training/{}/", pod_id);
let response = self
.client
.list_objects_v2()
.bucket(&self.bucket)
.prefix(&prefix)
.send()
.await
.map_err(|e| FoxhuntError::S3(format!("Failed to list objects: {}", e)))?;
let mut log_files = Vec::new();
let contents = response.contents();
for object in contents {
if let Some(key) = object.key() {
// Filter for .log files or stdout/stderr
if key.ends_with(".log")
|| key.ends_with("stdout")
|| key.ends_with("stderr")
{
log_files.push(key.to_owned());
}
}
}
// Sort by name (typically includes timestamp)
log_files.sort();
Ok(log_files)
}
/// Download a specific log file
pub(crate) async fn download_log(&self, path: &str) -> Result<String> {
let response = self
.client
.get_object()
.bucket(&self.bucket)
.key(path)
.send()
.await
.map_err(|e| FoxhuntError::S3(format!("Failed to get object {}: {}", path, e)))?;
let bytes = response
.body
.collect()
.await
.map_err(|e| FoxhuntError::S3(format!("Failed to read object body: {}", e)))?;
let content = String::from_utf8(bytes.to_vec())
.map_err(|e| FoxhuntError::S3(format!("Invalid UTF-8 in log file: {}", e)))?;
Ok(content)
}
/// Get the size of a log file in bytes
pub(crate) async fn get_log_size(&self, path: &str) -> Result<Option<i64>> {
match self
.client
.head_object()
.bucket(&self.bucket)
.key(path)
.send()
.await
{
Ok(response) => Ok(response.content_length()),
Err(e) => {
let err_str = format!("{:?}", e);
if err_str.contains("NotFound") || err_str.contains("404") {
Ok(None)
} else {
Err(FoxhuntError::S3(format!(
"Failed to get object metadata: {}",
e
)))
}
}
}
}
/// Download a range of bytes from a log file
pub(crate) async fn download_log_range(&self, path: &str, start: i64, end: i64) -> Result<String> {
let range = format!("bytes={}-{}", start, end);
let response = self
.client
.get_object()
.bucket(&self.bucket)
.key(path)
.range(range)
.send()
.await
.map_err(|e| {
FoxhuntError::S3(format!("Failed to get object range {}: {}", path, e))
})?;
let bytes = response
.body
.collect()
.await
.map_err(|e| FoxhuntError::S3(format!("Failed to read object body: {}", e)))?;
let content = String::from_utf8(bytes.to_vec())
.map_err(|e| FoxhuntError::S3(format!("Invalid UTF-8 in log file: {}", e)))?;
Ok(content)
}
}

253
foxhunt-deploy/src/s3/monitor.rs
View File

@@ -1,253 +0,0 @@
use super::parser::{colorize_log_line, detect_completion, filter_line, get_last_n_lines};
use super::S3LogClient;
use crate::error::{FoxhuntError, Result};
use colored::Colorize;
use std::time::Duration;
use tokio::time::sleep;
/// Log monitor for streaming S3 logs
pub(crate) struct LogMonitor {
client: S3LogClient,
pod_id: String,
poll_interval: Duration,
}
impl LogMonitor {
/// Create a new log monitor
pub(crate) fn new(client: S3LogClient, pod_id: String, poll_interval_secs: u64) -> Self {
Self {
client,
pod_id,
poll_interval: Duration::from_secs(poll_interval_secs),
}
}
/// Find the primary log file for a pod
async fn find_log_file(&self) -> Result<Option<String>> {
let log_files = self.client.list_log_files(&self.pod_id).await?;
if log_files.is_empty() {
return Ok(None);
}
// Priority: training.log > stdout > stderr > other .log files
let training_log = log_files.iter().find(|f| f.ends_with("training.log"));
if let Some(log) = training_log {
return Ok(Some(log.clone()));
}
let stdout = log_files.iter().find(|f| f.ends_with("stdout"));
if let Some(log) = stdout {
return Ok(Some(log.clone()));
}
let stderr = log_files.iter().find(|f| f.ends_with("stderr"));
if let Some(log) = stderr {
return Ok(Some(log.clone()));
}
// Return first .log file
Ok(log_files.first().cloned())
}
/// Show recent logs (non-following mode)
pub(crate) async fn show_recent_logs(&self, tail: Option<usize>) -> Result<()> {
let log_file = self
.find_log_file()
.await?
.ok_or_else(|| FoxhuntError::S3(format!("No log files found for pod {}", self.pod_id)))?;
println!(
"{} {}",
"Found log file:".cyan().bold(),
log_file.dimmed()
);
let content = self.client.download_log(&log_file).await?;
let lines = if let Some(n) = tail {
get_last_n_lines(&content, n)
} else {
content.lines().map(|s| s.to_owned()).collect()
};
for line in lines {
println!("{}", colorize_log_line(&line));
}
Ok(())
}
/// Tail logs in real-time (following mode)
#[allow(clippy::non_ascii_literal)]
pub(crate) async fn tail_logs(&self, tail: Option<usize>, filter: Option<String>) -> Result<()> {
println!(
"{} {}",
"Monitoring pod:".cyan().bold(),
self.pod_id.yellow()
);
println!(
"{} {}",
"Poll interval:".cyan().bold(),
format!("{}s", self.poll_interval.as_secs()).yellow()
);
if let Some(ref pattern) = filter {
println!(
"{} {}",
"Filter pattern:".cyan().bold(),
pattern.yellow()
);
}
println!("{}", "─".repeat(80).dimmed());
let log_file = loop {
match self.find_log_file().await {
Ok(Some(file)) => {
println!(
"{} {}",
"Found log file:".green().bold(),
file.dimmed()
);
break file;
}
Ok(None) => {
println!(
"{} (retrying in {}s...)",
"No log files found yet".yellow(),
self.poll_interval.as_secs()
);
sleep(self.poll_interval).await;
}
Err(e) => {
println!("{} {} (retrying...)", "Error:".red().bold(), e);
sleep(self.poll_interval).await;
}
}
};
println!("{}", "─".repeat(80).dimmed());
// Track last read position
let mut last_position: i64 = 0;
// Show initial tail lines if requested
if let Some(n) = tail {
match self.client.download_log(&log_file).await {
Ok(content) => {
let lines = get_last_n_lines(&content, n);
for line in &lines {
if filter_line(line, filter.as_deref()) {
println!("{}", colorize_log_line(line));
}
}
// Update position to end of file
last_position = content.len() as i64;
}
Err(e) => {
println!("{} {}", "Error reading initial logs:".yellow(), e);
}
}
}
// Main monitoring loop
let mut retry_count = 0;
const MAX_RETRIES: u32 = 3;
loop {
// Check current file size
match self.client.get_log_size(&log_file).await {
Ok(Some(current_size)) => {
retry_count = 0; // Reset on success
if current_size > last_position {
// New content available
match self
.client
.download_log_range(&log_file, last_position, current_size - 1)
.await
{
Ok(new_content) => {
let lines: Vec<&str> = new_content.lines().collect();
for line in lines {
if filter_line(line, filter.as_deref()) {
println!("{}", colorize_log_line(line));
// Check for completion
if detect_completion(line) {
println!("{}", "─".repeat(80).dimmed());
println!(
"{} {}",
"✓".green().bold(),
"Training completed!".green().bold()
);
return Ok(());
}
}
}
last_position = current_size;
}
Err(e) => {
println!(
"{} {}",
"Error reading new content:".yellow(),
e
);
}
}
}
}
Ok(None) => {
println!(
"{} (file may have been deleted)",
"Log file not found".yellow()
);
}
Err(e) => {
retry_count += 1;
println!(
"{} {} (retry {}/{})",
"Error checking file size:".yellow(),
e,
retry_count,
MAX_RETRIES
);
if retry_count >= MAX_RETRIES {
return Err(FoxhuntError::S3(format!(
"Failed to monitor logs after {} retries",
MAX_RETRIES
)));
}
}
}
// Wait before next poll
sleep(self.poll_interval).await;
}
}
/// List all available log files for the pod
#[allow(clippy::non_ascii_literal)]
pub(crate) async fn list_logs(&self) -> Result<()> {
println!(
"{} {}",
"Searching for logs for pod:".cyan().bold(),
self.pod_id.yellow()
);
let log_files = self.client.list_log_files(&self.pod_id).await?;
if log_files.is_empty() {
println!("{}", "No log files found".yellow());
return Ok(());
}
println!("{}", "─".repeat(80).dimmed());
println!("{} log file(s):", log_files.len());
for file in log_files {
println!(" {} {}", "•".cyan(), file);
}
Ok(())
}
}

214
foxhunt-deploy/src/s3/parser.rs
View File

@@ -1,214 +0,0 @@
use colored::Colorize;
use regex::Regex;
/// Log level detected in log lines
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum LogLevel {
Info,
Warn,
Error,
Debug,
Trace,
Unknown,
}
impl LogLevel {
/// Detect log level from a line
pub(crate) fn detect(line: &str) -> Self {
let upper = line.to_uppercase();
if upper.contains("ERROR") || upper.contains("FATAL") || upper.contains("CRITICAL") {
LogLevel::Error
} else if upper.contains("WARN") || upper.contains("WARNING") {
LogLevel::Warn
} else if upper.contains("INFO") {
LogLevel::Info
} else if upper.contains("DEBUG") {
LogLevel::Debug
} else if upper.contains("TRACE") {
LogLevel::Trace
} else {
LogLevel::Unknown
}
}
}
/// Colorize a log line based on its level
pub(crate) fn colorize_log_line(line: &str) -> String {
let level = LogLevel::detect(line);
match level {
LogLevel::Error => line.red().to_string(),
LogLevel::Warn => line.yellow().to_string(),
LogLevel::Info => line.green().to_string(),
LogLevel::Debug => line.cyan().to_string(),
LogLevel::Trace => line.dimmed().to_string(),
LogLevel::Unknown => line.to_owned(),
}
}
/// Training metrics extracted from log lines
#[derive(Debug, Clone, Default)]
pub(crate) struct TrainingMetrics {
pub epoch: Option<u32>,
pub loss: Option<f64>,
pub accuracy: Option<f64>,
pub learning_rate: Option<f64>,
}
/// Parse training metrics from a log line
#[allow(dead_code)]
pub(crate) fn parse_training_metrics(line: &str) -> Option<TrainingMetrics> {
let mut metrics = TrainingMetrics::default();
let mut found_any = false;
// Epoch patterns
let epoch_patterns = [
Regex::new(r"epoch[:\s]+(\d+)").ok()?,
Regex::new(r"Epoch[:\s]+(\d+)").ok()?,
Regex::new(r"EPOCH[:\s]+(\d+)").ok()?,
];
for pattern in &epoch_patterns {
if let Some(caps) = pattern.captures(line) {
if let Some(epoch_str) = caps.get(1) {
if let Ok(epoch) = epoch_str.as_str().parse::<u32>() {
metrics.epoch = Some(epoch);
found_any = true;
break;
}
}
}
}
// Loss patterns
let loss_patterns = [
Regex::new(r"loss[:\s]+([0-9.]+)").ok()?,
Regex::new(r"Loss[:\s]+([0-9.]+)").ok()?,
Regex::new(r"LOSS[:\s]+([0-9.]+)").ok()?,
];
for pattern in &loss_patterns {
if let Some(caps) = pattern.captures(line) {
if let Some(loss_str) = caps.get(1) {
if let Ok(loss) = loss_str.as_str().parse::<f64>() {
metrics.loss = Some(loss);
found_any = true;
break;
}
}
}
}
// Learning rate patterns
let lr_patterns = [
Regex::new(r"lr[:\s]+([0-9.e-]+)").ok()?,
Regex::new(r"learning_rate[:\s]+([0-9.e-]+)").ok()?,
];
for pattern in &lr_patterns {
if let Some(caps) = pattern.captures(line) {
if let Some(lr_str) = caps.get(1) {
if let Ok(lr) = lr_str.as_str().parse::<f64>() {
metrics.learning_rate = Some(lr);
found_any = true;
break;
}
}
}
}
found_any.then_some(metrics)
}
/// Detect if a log line indicates training completion
pub(crate) fn detect_completion(line: &str) -> bool {
let lower = line.to_lowercase();
lower.contains("training complete")
|| lower.contains("training finished")
|| lower.contains("training done")
|| lower.contains("saved final model")
|| lower.contains("checkpoint saved")
|| (lower.contains("epoch") && lower.contains("/") && lower.contains("100%"))
}
/// Filter a log line by regex pattern
pub(crate) fn filter_line(line: &str, pattern_str: Option<&str>) -> bool {
match pattern_str {
None => true,
Some(pat) => {
if let Ok(regex) = Regex::new(pat) {
regex.is_match(line)
} else {
// Invalid regex, show all lines
true
}
}
}
}
/// Get last N lines from log content
pub(crate) fn get_last_n_lines(content: &str, n: usize) -> Vec<String> {
let lines: Vec<String> = content.lines().map(|s| s.to_owned()).collect();
let start_index = lines.len().saturating_sub(n);
lines[start_index..].to_vec()
}
/// Format a summary of training metrics
#[allow(dead_code)]
pub(crate) fn format_metrics_summary(metrics: &TrainingMetrics) -> String {
let mut parts = Vec::new();
if let Some(epoch) = metrics.epoch {
parts.push(format!("Epoch: {}", epoch));
}
if let Some(loss) = metrics.loss {
parts.push(format!("Loss: {:.6}", loss));
}
if let Some(acc) = metrics.accuracy {
parts.push(format!("Acc: {:.4}", acc));
}
if let Some(lr) = metrics.learning_rate {
parts.push(format!("LR: {:.6}", lr));
}
if parts.is_empty() {
"No metrics".to_owned()
} else {
parts.join(" | ")
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_detect_log_level() {
assert_eq!(LogLevel::detect("ERROR: Something failed"), LogLevel::Error);
assert_eq!(LogLevel::detect("WARN: Low memory"), LogLevel::Warn);
assert_eq!(LogLevel::detect("INFO: Starting training"), LogLevel::Info);
}
#[test]
fn test_parse_training_metrics() {
let line = "INFO: Epoch: 10, Loss: 0.345, lr: 0.001";
let metrics = parse_training_metrics(line).unwrap();
assert_eq!(metrics.epoch, Some(10));
assert_eq!(metrics.loss, Some(0.345));
assert_eq!(metrics.learning_rate, Some(0.001));
}
#[test]
fn test_detect_completion() {
assert!(detect_completion("Training complete! Model saved."));
assert!(detect_completion("Checkpoint saved at epoch 100"));
assert!(!detect_completion("Starting training..."));
}
#[test]
fn test_get_last_n_lines() {
let content = "line1\nline2\nline3\nline4\nline5";
let last_3 = get_last_n_lines(content, 3);
assert_eq!(last_3, vec!["line3", "line4", "line5"]);
}
}

4
foxhunt-deploy/src/utils/mod.rs
View File

@@ -1,4 +0,0 @@
pub(crate) mod terminal;
// Re-export terminal functions for convenience
pub(crate) use terminal::{error, info, step, success};

50
foxhunt-deploy/src/utils/terminal.rs
View File

@@ -1,50 +0,0 @@
use colored::Colorize;
/// Print success message with green checkmark
#[allow(clippy::non_ascii_literal)]
pub(crate) fn success(msg: impl AsRef<str>) {
println!("{} {}", "✓".green().bold(), msg.as_ref());
}
/// Print error message with red X
#[allow(clippy::non_ascii_literal)]
pub(crate) fn error(msg: impl AsRef<str>) {
eprintln!("{} {}", "✗".red().bold(), msg.as_ref());
}
/// Print info message with blue icon
#[allow(clippy::non_ascii_literal)]
pub(crate) fn info(msg: impl AsRef<str>) {
println!("{} {}", "ℹ".blue().bold(), msg.as_ref());
}
/// Print warning message with yellow icon
#[allow(dead_code)]
#[allow(clippy::non_ascii_literal)]
pub(crate) fn warning(msg: impl AsRef<str>) {
println!("{} {}", "⚠".yellow().bold(), msg.as_ref());
}
/// Print numbered step progress
pub(crate) fn step(step: usize, total: usize, msg: impl AsRef<str>) {
println!(
"{} {}",
format!("[{}/{}]", step, total).cyan().bold(),
msg.as_ref()
);
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_terminal_functions() {
// Just verify these don't panic
success("Test success");
error("Test error");
info("Test info");
warning("Test warning");
step(1, 5, "Test step");
}
}

169
foxhunt-deploy/tests/docker_integration_tests.rs
View File

@@ -1,169 +0,0 @@
use assert_cmd::Command;
use predicates::prelude::*;
#[test]
fn test_build_command_help() {
let mut cmd = Command::cargo_bin("foxhunt-deploy").unwrap();
cmd.arg("build").arg("--help");
cmd.assert()
.success()
.stdout(predicate::str::contains("Build Docker image"));
}
#[test]
fn test_build_requires_config() {
// Test that build command requires valid config
let mut cmd = Command::cargo_bin("foxhunt-deploy").unwrap();
cmd.arg("--config")
.arg("/tmp/nonexistent-config.toml")
.arg("build");
cmd.assert()
.failure()
.stderr(predicate::str::contains("Config file not found"));
}
#[test]
fn test_build_validates_dockerfile_existence() {
// This test verifies that the build command would check for Dockerfile existence
// Note: We can't actually run docker build in CI without Docker installed
// This is a documentation test showing expected behavior
let result = std::panic::catch_unwind(|| {
// If Docker is not available, the verify_docker_available() will fail
// This is the expected behavior
});
assert!(result.is_ok());
}
#[test]
fn test_build_options_builder_pattern() {
// Test the DockerBuildOptions builder pattern (unit test style)
// This verifies the API design is correct
// Note: We import via the binary's exposed API
// In a real scenario, these would be re-exported from lib.rs
// For now, this documents the expected API
let expected_tag = "test:latest";
let expected_dockerfile = "Dockerfile.test";
let expected_context = "/tmp/test";
// Verify our design expectations
assert_eq!(expected_tag, "test:latest");
assert_eq!(expected_dockerfile, "Dockerfile.test");
assert_eq!(expected_context, "/tmp/test");
}
#[test]
fn test_build_error_handling_no_docker() {
// This test documents expected behavior when Docker is not installed
// The actual error would be: "Docker is not installed"
let error_message = "Docker is not installed. Please install Docker Desktop or Docker Engine.";
assert!(error_message.contains("Docker"));
assert!(error_message.contains("install"));
}
#[test]
fn test_build_error_handling_daemon_not_running() {
// This test documents expected behavior when Docker daemon is not running
// The actual error would be: "Docker daemon is not running"
let error_message = "Docker daemon is not running. Please start Docker Desktop or Docker service.";
assert!(error_message.contains("daemon"));
assert!(error_message.contains("not running"));
}
#[test]
fn test_push_error_handling_no_auth() {
// This test documents expected behavior when registry auth fails
// The actual error would contain "authentication" or "unauthorized"
let error_message = "Docker registry authentication failed. Please run 'docker login' first.";
assert!(error_message.contains("authentication"));
assert!(error_message.contains("docker login"));
}
#[test]
fn test_push_error_handling_image_not_found() {
// This test documents expected behavior when image doesn't exist
let error_message = "Image 'test:latest' does not exist locally. Build it first.";
assert!(error_message.contains("does not exist"));
assert!(error_message.contains("Build it first"));
}
#[test]
fn test_build_args_defaults() {
// Test CLI argument defaults
let mut cmd = Command::cargo_bin("foxhunt-deploy").unwrap();
cmd.arg("build").arg("--help");
cmd.assert()
.success()
.stdout(predicate::str::contains("default: latest"))
.stdout(predicate::str::contains("Dockerfile.foxhunt-build"));
}
#[test]
fn test_build_with_no_push_flag() {
// Test that --no-push flag is recognized
let mut cmd = Command::cargo_bin("foxhunt-deploy").unwrap();
cmd.arg("build").arg("--help");
cmd.assert()
.success()
.stdout(predicate::str::contains("no-push"))
.stdout(predicate::str::contains("Skip pushing"));
}
#[test]
fn test_build_with_no_cache_flag() {
// Test that --no-cache flag is recognized
let mut cmd = Command::cargo_bin("foxhunt-deploy").unwrap();
cmd.arg("build").arg("--help");
cmd.assert()
.success()
.stdout(predicate::str::contains("no-cache"))
.stdout(predicate::str::contains("Disable Docker build cache"));
}
#[test]
fn test_build_with_custom_tag() {
// Test that custom tag argument works
let mut cmd = Command::cargo_bin("foxhunt-deploy").unwrap();
cmd.arg("build").arg("--help");
cmd.assert()
.success()
.stdout(predicate::str::contains("--tag"))
.stdout(predicate::str::contains("Docker image tag"));
}
#[test]
fn test_build_with_custom_dockerfile() {
// Test that custom Dockerfile argument works
let mut cmd = Command::cargo_bin("foxhunt-deploy").unwrap();
cmd.arg("build").arg("--help");
cmd.assert()
.success()
.stdout(predicate::str::contains("--dockerfile"))
.stdout(predicate::str::contains("Dockerfile path"));
}
#[test]
fn test_build_with_custom_context() {
// Test that custom context argument works
let mut cmd = Command::cargo_bin("foxhunt-deploy").unwrap();
cmd.arg("build").arg("--help");
cmd.assert()
.success()
.stdout(predicate::str::contains("--context"))
.stdout(predicate::str::contains("Docker build context path"));
}

0
tli/CONFIG_FILE_SUPPORT.md → fxt/CONFIG_FILE_SUPPORT.md
View File

8
tli/Cargo.toml → fxt/Cargo.toml
View File

@@ -1,5 +1,5 @@
[package]
name = "tli"
name = "fxt"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
@@ -11,11 +11,11 @@ documentation.workspace = true
publish.workspace = true
keywords.workspace = true
categories.workspace = true
description = "Terminal Line Interface for Foxhunt HFT Trading System"
description = "Foxhunt CLI — command-line interface for Foxhunt HFT Trading System"
# TLI binary - client-only terminal interface
# CLI binary
[[bin]]
name = "tli"
name = "fxt"
path = "src/main.rs"
[dependencies]

0
tli/TUNE_COMMAND_README.md → fxt/TUNE_COMMAND_README.md
View File

14
tli/benches/client_performance.rs → fxt/benches/client_performance.rs
View File

@@ -23,8 +23,8 @@
// DISABLED: use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
// DISABLED: use std::time::Duration;
// DISABLED: use tli::prelude::*;
// DISABLED: use tli::{ServiceEndpoints, TliClient};
// DISABLED: use fxt::prelude::*;
// DISABLED: use fxt::{ServiceEndpoints, TliClient};
// DISABLED: use tokio::runtime::Runtime;
/*
@@ -100,7 +100,7 @@ fn bench_endpoint_parsing(c: &mut Criterion) {
fn bench_request_serialization(c: &mut Criterion) {
let mut group = c.benchmark_group("request_serialization");
use tli::proto::trading::*;
use fxt::proto::trading::*;
// Order submission request
group.bench_function("submit_order_request", |b| {
@@ -166,7 +166,7 @@ fn bench_request_serialization(c: &mut Criterion) {
fn bench_response_deserialization(c: &mut Criterion) {
let mut group = c.benchmark_group("response_deserialization");
use tli::proto::trading::*;
use fxt::proto::trading::*;
// Order response
let order_response_json = r#"{
@@ -350,7 +350,7 @@ fn bench_memory_patterns(c: &mut Criterion) {
// Large response handling
group.bench_function("large_response_handling", |b| {
use tli::proto::trading::*;
use fxt::proto::trading::*;
b.iter(|| {
let mut orders = Vec::new();
@@ -379,7 +379,7 @@ fn bench_memory_patterns(c: &mut Criterion) {
// Streaming data structures
group.bench_function("streaming_data_structures", |b| {
use tli::proto::trading::*;
use fxt::proto::trading::*;
b.iter(|| {
let mut updates = Vec::new();
@@ -457,7 +457,7 @@ fn bench_config_operations(c: &mut Criterion) {
fn bench_health_check_operations(c: &mut Criterion) {
let mut group = c.benchmark_group("health_check_operations");
use tli::client::ServiceHealth;
use fxt::client::ServiceHealth;
// Health status creation
group.bench_function("health_status_creation", |b| {

6
tli/benches/configuration_benchmarks.rs → fxt/benches/configuration_benchmarks.rs
View File

@@ -20,8 +20,8 @@
// DISABLED: use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
// DISABLED: use std::collections::HashMap;
// DISABLED: use std::time::Duration;
// DISABLED: use tli::prelude::*;
// DISABLED: use tli::types::*;
// DISABLED: use fxt::prelude::*;
// DISABLED: use fxt::types::*;
// DISABLED: use tokio::runtime::Runtime;
/*
@@ -140,7 +140,7 @@ fn bench_type_conversions(c: &mut Criterion) {
let mut group = c.benchmark_group("type_conversions");
use common::OrderSide;
use tli::proto::trading::{OrderStatus, OrderType};
use fxt::proto::trading::{OrderStatus, OrderType};
// Order side conversions
let sides = vec![OrderSide::Buy, OrderSide::Sell];

4
tli/benches/encryption_performance.rs → fxt/benches/encryption_performance.rs
View File

@@ -1,5 +1,5 @@
use criterion::{black_box, criterion_group, criterion_main, Criterion};
use tli::auth::token_manager::{FileTokenStorage, TokenStorage};
use fxt::auth::token_manager::{FileTokenStorage, TokenStorage};
fn create_test_storage() -> FileTokenStorage {
// Use default directory - benchmark will measure real-world performance
@@ -56,7 +56,7 @@ fn bench_roundtrip_encrypted(c: &mut Criterion) {
}
fn bench_key_derivation(c: &mut Criterion) {
use tli::auth::KeyManager;
use fxt::auth::KeyManager;
c.bench_function("key_derivation_first_call", |b| {
b.iter(|| {

6
tli/benches/serialization_benchmarks.rs → fxt/benches/serialization_benchmarks.rs
View File

@@ -14,12 +14,12 @@
//! - `MetricValue` - proto has Metric instead
//!
//! TODO: Fix this benchmark by either:
//! 1. Adding the missing proto message definitions to tli/proto/trading.proto
//! 1. Adding the missing proto message definitions to fxt/proto/trading.proto
//! 2. Rewriting benchmarks to use existing proto types (GetOrderStatusResponse, OrderUpdateEvent, Metric)
//! 3. Creating separate benchmark-specific proto messages
//!
//! Related files:
//! - tli/proto/trading.proto - contains actual protobuf definitions
//! - fxt/proto/trading.proto - contains actual protobuf definitions
//! - tli/build.rs - compiles proto files to Rust code
//!
//! See Wave 36 - Agent 1 for context
@@ -30,7 +30,7 @@
// DISABLED: use prost::Message;
// DISABLED: use std::collections::HashMap;
// DISABLED: use std::time::Duration;
// DISABLED: use tli::proto::trading::*;
// DISABLED: use fxt::proto::trading::*;
/*
/// Benchmark protobuf serialization

0
tli/build.rs → fxt/build.rs
View File

0
tli/config.toml.example → fxt/config.toml.example
View File

0
tli/docs/AUTHENTICATION.md → fxt/docs/AUTHENTICATION.md
View File

0
tli/docs/ML_TRADING_COMMANDS.md → fxt/docs/ML_TRADING_COMMANDS.md
View File

0
tli/docs/USAGE.md → fxt/docs/USAGE.md
View File

2
tli/examples/complete_client_example.rs → fxt/examples/complete_client_example.rs
View File

@@ -4,7 +4,7 @@
//! to connect to both Trading Service and Backtesting Service.
#![allow(unused_crate_dependencies)]
use tli::prelude::*;
use fxt::prelude::*;
use tokio::time::{sleep, Duration};
use tracing::{error, info, warn};

0
tli/examples/config_demo.rs → fxt/examples/config_demo.rs
View File

0
tli/examples/config_management_placeholder.rs → fxt/examples/config_management_placeholder.rs
View File

0
tli/examples/security_example.rs → fxt/examples/security_example.rs
View File

0
tli/proptest-regressions/tests.txt → fxt/proptest-regressions/tests.txt
View File

0
tli/proto/config.proto → fxt/proto/config.proto
View File

0
tli/proto/health.proto → fxt/proto/health.proto
View File

0
tli/proto/ml.proto → fxt/proto/ml.proto
View File

0
tli/proto/ml_training.proto → fxt/proto/ml_training.proto
View File

0
tli/proto/trading.proto → fxt/proto/trading.proto
View File

0
tli/proto/trading_agent.proto → fxt/proto/trading_agent.proto
View File

12
tli/src/auth/encryption.rs → fxt/src/auth/encryption.rs
View File

@@ -42,7 +42,7 @@ impl EncryptionFormat {
///
/// # Examples
/// ```
/// use tli::auth::encryption::EncryptionFormat;
/// use fxt::auth::encryption::EncryptionFormat;
///
/// let hex_data = "4a5b6c7d8e9f0a1b";
/// assert_eq!(EncryptionFormat::detect(hex_data), EncryptionFormat::HexEncoded);
@@ -68,7 +68,7 @@ impl EncryptionFormat {
///
/// # Examples
/// ```
/// use tli::auth::encryption::EncryptionFormat;
/// use fxt::auth::encryption::EncryptionFormat;
///
/// assert!(!EncryptionFormat::is_encrypted("4a5b6c7d"));
/// assert!(EncryptionFormat::is_encrypted("ENC:data"));
@@ -101,7 +101,7 @@ impl EncryptionFormat {
///
/// # Examples
/// ```no_run
/// use tli::auth::encryption::encrypt_token;
/// use fxt::auth::encryption::encrypt_token;
///
/// let key = [0u8; 32]; // 32-byte key (in production, use proper key derivation)
/// let token = "my_secret_token";
@@ -180,7 +180,7 @@ pub fn encrypt_token(token: &str, key: &[u8]) -> Result<String, CommonError> {
///
/// # Examples
/// ```no_run
/// use tli::auth::encryption::{encrypt_token, decrypt_token};
/// use fxt::auth::encryption::{encrypt_token, decrypt_token};
///
/// let key = [0u8; 32];
/// let token = "my_secret_token";
@@ -279,7 +279,7 @@ pub fn decrypt_token(encrypted: &str, key: &[u8]) -> Result<String, CommonError>
///
/// # Examples
/// ```no_run
/// use tli::auth::encryption::read_token_auto;
/// use fxt::auth::encryption::read_token_auto;
///
/// let key = [0u8; 32];
///
@@ -338,7 +338,7 @@ pub fn read_token_auto(encrypted_data: &str, key: &[u8]) -> Result<String, Commo
///
/// # Examples
/// ```no_run
/// use tli::auth::encryption::{read_token_auto, write_token_encrypted};
/// use fxt::auth::encryption::{read_token_auto, write_token_encrypted};
///
/// let key = [0u8; 32];
///

0
tli/src/auth/interceptor.rs → fxt/src/auth/interceptor.rs
View File

0
tli/src/auth/jwt_generator.rs → fxt/src/auth/jwt_generator.rs
View File

0
tli/src/auth/key_manager.rs → fxt/src/auth/key_manager.rs
View File

0
tli/src/auth/login.rs → fxt/src/auth/login.rs
View File

0
tli/src/auth/mod.rs → fxt/src/auth/mod.rs
View File

0
tli/src/auth/token_manager.rs → fxt/src/auth/token_manager.rs
View File

0
tli/src/client/backtesting_client.rs → fxt/src/client/backtesting_client.rs
View File

0
tli/src/client/connection_manager.rs → fxt/src/client/connection_manager.rs
View File

0
tli/src/client/ml_training_client.rs → fxt/src/client/ml_training_client.rs
View File

0
tli/src/client/mod.rs → fxt/src/client/mod.rs
View File

0
tli/src/client/trading_client.rs → fxt/src/client/trading_client.rs
View File

0
tli/src/commands/agent.rs → fxt/src/commands/agent.rs
View File

0
tli/src/commands/auth.rs → fxt/src/commands/auth.rs
View File

0
tli/src/commands/backtest_ml.rs → fxt/src/commands/backtest_ml.rs
View File

0
tli/src/commands/mod.rs → fxt/src/commands/mod.rs
View File

4
tli/src/commands/trade.rs → fxt/src/commands/trade.rs
View File

@@ -51,8 +51,8 @@ pub enum TradeCommand {
///
/// # Example
/// ```no_run
/// use tli::commands::trade::{TradeArgs, TradeCommand, execute_trade_command};
/// use tli::commands::trade_ml::TradeMlArgs;
/// use fxt::commands::trade::{TradeArgs, TradeCommand, execute_trade_command};
/// use fxt::commands::trade_ml::TradeMlArgs;
///
/// # async fn example() -> anyhow::Result<()> {
/// let args = TradeArgs {

0
tli/src/commands/trade_ml.rs → fxt/src/commands/trade_ml.rs
View File

0
tli/src/commands/train/list.rs → fxt/src/commands/train/list.rs
View File

0
tli/src/commands/train/mod.rs → fxt/src/commands/train/mod.rs
View File

0
tli/src/commands/train/progress_tracker.rs → fxt/src/commands/train/progress_tracker.rs
View File

0
tli/src/commands/train/status.rs → fxt/src/commands/train/status.rs
View File

0
tli/src/commands/tune.rs → fxt/src/commands/tune.rs
View File

0
tli/src/commands/tune_impl.rs → fxt/src/commands/tune_impl.rs
View File

0
tli/src/commands/tune_stream.rs → fxt/src/commands/tune_stream.rs
View File

4
tli/src/config.rs → fxt/src/config.rs
View File

@@ -69,7 +69,7 @@ impl TliConfig {
///
/// # Examples
/// ```no_run
/// use tli::config::TliConfig;
/// use fxt::config::TliConfig;
///
/// let config = TliConfig::load().unwrap();
/// println!("API Gateway: {}", config.api_gateway_url);
@@ -96,7 +96,7 @@ impl TliConfig {
///
/// # Examples
/// ```no_run
/// use tli::config::TliConfig;
/// use fxt::config::TliConfig;
///
/// let config = TliConfig {
/// api_gateway_url: "http://localhost:50051".to_string(),

0
tli/src/error.rs → fxt/src/error.rs
View File

2
tli/src/lib.rs → fxt/src/lib.rs
View File

@@ -30,7 +30,7 @@
//!
//! ## Example Usage
//! ```rust,no_run
//! use tli::prelude::*;
//! use fxt::prelude::*;
//!
//! #[tokio::main]
//! async fn main() -> TliResult<()> {

62
tli/src/main.rs → fxt/src/main.rs
View File

@@ -11,8 +11,8 @@ use colored::Colorize;
use jsonwebtoken::{decode, Algorithm, DecodingKey, Validation};
use serde::{Deserialize, Serialize};
use std::time::{SystemTime, UNIX_EPOCH};
use tli::auth::token_manager::FileTokenStorage;
use tli::{
use fxt::auth::token_manager::FileTokenStorage;
use fxt::{
commands::{
agent::{execute_agent_command, AgentArgs},
auth::{execute_auth_command, AuthCommand},
@@ -68,7 +68,7 @@ use zeroize as _;
/// TLI Command Line Interface
#[derive(Parser)]
#[clap(
name = "tli",
name = "fxt",
version,
about = "Foxhunt Trading System Terminal Interface",
long_about = "Foxhunt Trading System Terminal Interface\n\n\
@@ -120,10 +120,10 @@ enum Commands {
Supported models: DQN, PPO, MAMBA_2, TFT, TLOB, LIQUID\n\
Uses Optuna for Bayesian optimization.\n\n\
Examples:\n\
tli tune start --model DQN --trials 100\n\
tli tune status --job-id <uuid>\n\
tli tune best --job-id <uuid>\n\
tli tune stop --job-id <uuid>"
fxt tune start --model DQN --trials 100\n\
fxt tune status --job-id <uuid>\n\
fxt tune best --job-id <uuid>\n\
fxt tune stop --job-id <uuid>"
)]
Tune {
#[clap(subcommand)]
@@ -139,11 +139,11 @@ enum Commands {
- Watch real-time training progress\n\
- Stop jobs gracefully or forcefully\n\n\
Examples:\n\
tli train list\n\
tli train list --status RUNNING --model TFT\n\
tli train status train_dqn_es_20251022_1430\n\
tli train watch train_tft_nq_20251022_1500\n\
tli train stop train_ppo_es_20251022_1600"
fxt train list\n\
fxt train list --status RUNNING --model TFT\n\
fxt train status train_dqn_es_20251022_1430\n\
fxt train watch train_tft_nq_20251022_1500\n\
fxt train stop train_ppo_es_20251022_1600"
)]
Train {
#[clap(subcommand)]
@@ -155,10 +155,10 @@ enum Commands {
JWT tokens are stored securely in OS keyring.\n\
Tokens auto-refresh when expiring (within 60 seconds).\n\n\
Examples:\n\
tli auth login --username trader1\n\
tli auth status\n\
tli auth refresh\n\
tli auth logout")]
fxt auth login --username trader1\n\
fxt auth status\n\
fxt auth refresh\n\
fxt auth logout")]
Auth {
#[clap(subcommand)]
auth_cmd: AuthCommand,
@@ -170,8 +170,8 @@ enum Commands {
Subcommands:\n\
allocate-portfolio - Allocate capital across selected assets\n\n\
Examples:\n\
tli agent allocate-portfolio --selection-id abc-123 --total-capital 100000\n\
tli agent allocate-portfolio --selection-id abc-123 --total-capital 100000 --strategy risk-parity"
fxt agent allocate-portfolio --selection-id abc-123 --total-capital 100000\n\
fxt agent allocate-portfolio --selection-id abc-123 --total-capital 100000 --strategy risk-parity"
)]
Agent {
#[command(flatten)]
@@ -223,8 +223,8 @@ struct Claims {
/// - `Ok(String)` - Valid access token (possibly refreshed)
/// - `Err(anyhow::Error)` - Token not found, refresh failed, or invalid format
async fn load_jwt_token(api_gateway_url: &str) -> Result<String> {
use tli::auth::login::LoginClient;
use tli::auth::token_manager::{AuthTokenManager, TokenStorage};
use fxt::auth::login::LoginClient;
use fxt::auth::token_manager::{AuthTokenManager, TokenStorage};
use tonic::transport::Channel;
let storage = FileTokenStorage::new().context("Failed to initialize file token storage")?;
@@ -242,7 +242,7 @@ async fn load_jwt_token(api_gateway_url: &str) -> Result<String> {
if storage.get_refresh_token().await?.is_none() {
anyhow::bail!(
"No refresh token available. Please login: {}",
"tli auth login".bright_cyan()
"fxt auth login".bright_cyan()
);
}
@@ -283,14 +283,14 @@ async fn load_jwt_token(api_gateway_url: &str) -> Result<String> {
if stored_refresh.is_empty() {
anyhow::bail!(
"Token refresh succeeded but refresh token is empty in keyring. Please login again: {}",
"tli auth login".bright_cyan()
"fxt auth login".bright_cyan()
)
}
},
None => {
anyhow::bail!(
"Token refresh succeeded but refresh token not found in keyring. Please login again: {}",
"tli auth login".bright_cyan()
"fxt auth login".bright_cyan()
)
},
}
@@ -301,7 +301,7 @@ async fn load_jwt_token(api_gateway_url: &str) -> Result<String> {
None => {
anyhow::bail!(
"Token refresh succeeded but new token not found in keyring. Please login again: {}",
"tli auth login".bright_cyan()
"fxt auth login".bright_cyan()
)
},
}
@@ -313,7 +313,7 @@ async fn load_jwt_token(api_gateway_url: &str) -> Result<String> {
None => {
anyhow::bail!(
"Not authenticated. Please run: {} first",
"tli auth login".bright_cyan()
"fxt auth login".bright_cyan()
)
},
}
@@ -426,7 +426,7 @@ mod tests {
fn test_cli_parsing_tune_command() {
// Test that Cli struct parses tune command correctly
let cli = Cli::parse_from(&[
"tli",
"fxt",
"--api-gateway-url",
"http://test.com",
"tune",
@@ -445,7 +445,7 @@ mod tests {
#[test]
fn test_cli_parsing_auth_command() {
let cli = Cli::parse_from(&["tli", "auth", "status"]);
let cli = Cli::parse_from(&["fxt", "auth", "status"]);
match cli.command {
Commands::Auth { .. } => {},
@@ -456,7 +456,7 @@ mod tests {
#[test]
fn test_cli_default_values() {
// Test default values when no flags provided
let cli = Cli::parse_from(&["tli", "auth", "status"]);
let cli = Cli::parse_from(&["fxt", "auth", "status"]);
assert_eq!(cli.api_gateway_url, "http://localhost:50051");
assert_eq!(cli.log_level, "info");
@@ -467,7 +467,7 @@ mod tests {
fn test_cli_custom_values() {
// Test that custom values override defaults
let cli = Cli::parse_from(&[
"tli",
"fxt",
"--api-gateway-url",
"http://custom.com:8080",
"--log-level",
@@ -486,7 +486,7 @@ mod tests {
#[test]
fn test_tune_command_with_all_args() {
let cli = Cli::parse_from(&[
"tli",
"fxt",
"tune",
"start",
"--model",
@@ -506,7 +506,7 @@ mod tests {
#[test]
fn test_auth_login_command_parsing() {
let cli = Cli::parse_from(&["tli", "auth", "login", "--username", "testuser"]);
let cli = Cli::parse_from(&["fxt", "auth", "login", "--username", "testuser"]);
match cli.command {
Commands::Auth { .. } => {},

0
tli/src/prelude.rs → fxt/src/prelude.rs
View File

0
tli/src/tests.rs → fxt/src/tests.rs
View File

0
tli/src/types.rs → fxt/src/types.rs
View File

0
tli/tests/INTEGRATION_TEST_GUIDE.md → fxt/tests/INTEGRATION_TEST_GUIDE.md
View File

0
tli/tests/TEST_EXECUTION_README.md → fxt/tests/TEST_EXECUTION_README.md
View File

Some files were not shown because too many files have changed in this diff Show More