e8a68ee39f
- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API - Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT) - Files saved to test_data/real/databento/ml_training/ - Total: 360 files, 15 MB compressed DBN format - Used existing Rust pattern from download_nq_fut.rs - API key loaded from .env file - 100% success rate (360/360 files) - Ready for ML training benchmarks Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
172 lines
6.5 KiB
Plaintext
172 lines
6.5 KiB
Plaintext
# Environment Configuration Template for Foxhunt HFT System
|
|
# Copy this file to .env and configure with your actual values
|
|
|
|
# =============================================================================
|
|
# JWT Authentication Configuration (REQUIRED)
|
|
# =============================================================================
|
|
|
|
# JWT Secret - MUST be set for authentication to work
|
|
# Generate with: openssl rand -base64 96
|
|
JWT_SECRET=your_jwt_secret_here_change_in_production
|
|
JWT_ISSUER=foxhunt-trading
|
|
JWT_AUDIENCE=trading-api
|
|
|
|
# =============================================================================
|
|
# AWS S3 Configuration
|
|
# =============================================================================
|
|
|
|
# AWS Credentials - Required for S3 storage functionality
|
|
AWS_ACCESS_KEY_ID=your_aws_access_key_id
|
|
AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key
|
|
AWS_REGION=us-east-1
|
|
|
|
# S3 Bucket Configuration
|
|
S3_ARCHIVAL_BUCKET=foxhunt-archives
|
|
S3_MODEL_STORAGE_BUCKET=foxhunt-models
|
|
S3_CHECKPOINT_BUCKET=foxhunt-checkpoints
|
|
|
|
# S3 Storage Classes (cost optimization)
|
|
# Options: STANDARD, STANDARD_IA, ONEZONE_IA, REDUCED_REDUNDANCY, GLACIER, DEEP_ARCHIVE
|
|
S3_DEFAULT_STORAGE_CLASS=STANDARD_IA
|
|
S3_COMPLIANCE_STORAGE_CLASS=GLACIER
|
|
|
|
# =============================================================================
|
|
# HashiCorp Vault Configuration (Recommended for Production)
|
|
# =============================================================================
|
|
|
|
# Vault server configuration
|
|
VAULT_ADDR=http://localhost:8200
|
|
VAULT_TOKEN=your_vault_token
|
|
|
|
# Vault paths for S3 credentials (secure storage)
|
|
VAULT_S3_CREDENTIALS_PATH=secret/data/foxhunt/s3
|
|
VAULT_AWS_CREDENTIALS_PATH=secret/data/foxhunt/aws
|
|
|
|
# =============================================================================
|
|
# Database Configuration
|
|
# =============================================================================
|
|
|
|
DATABASE_URL=postgresql://localhost/foxhunt
|
|
REDIS_URL=redis://localhost:6379
|
|
|
|
# =============================================================================
|
|
# Trading System Configuration
|
|
# =============================================================================
|
|
|
|
# Environment type (development, staging, production)
|
|
ENVIRONMENT=development
|
|
|
|
# Trading configuration
|
|
FOXHUNT_REDIS_URL=redis://localhost:6379
|
|
REDIS_HOST=localhost
|
|
REDIS_PORT=6379
|
|
|
|
# =============================================================================
|
|
# Logging and Monitoring
|
|
# =============================================================================
|
|
|
|
RUST_LOG=info
|
|
RUST_BACKTRACE=1
|
|
|
|
# =============================================================================
|
|
# S3 Archival Settings
|
|
# =============================================================================
|
|
|
|
# Data retention policies (in days)
|
|
S3_ARCHIVAL_RETENTION_DAYS=2555 # 7 years for regulatory compliance
|
|
S3_MARKET_DATA_RETENTION_DAYS=1825 # 5 years
|
|
S3_TRADING_LOGS_RETENTION_DAYS=2555 # 7 years
|
|
S3_RISK_LOGS_RETENTION_DAYS=2555 # 7 years
|
|
|
|
# Compression settings
|
|
S3_ENABLE_COMPRESSION=true
|
|
S3_COMPRESSION_ALGORITHM=gzip # Options: gzip, lz4, zstd
|
|
|
|
# Lifecycle management
|
|
S3_ENABLE_LIFECYCLE=true
|
|
S3_TRANSITION_TO_IA_DAYS=30 # Move to Infrequent Access after 30 days
|
|
S3_TRANSITION_TO_GLACIER_DAYS=90 # Move to Glacier after 90 days
|
|
S3_TRANSITION_TO_DEEP_ARCHIVE_DAYS=365 # Move to Deep Archive after 1 year
|
|
|
|
# =============================================================================
|
|
# Security Settings
|
|
# =============================================================================
|
|
|
|
# S3 Encryption
|
|
S3_ENABLE_ENCRYPTION=true
|
|
S3_ENCRYPTION_ALGORITHM=AES256 # Options: AES256, aws:kms
|
|
|
|
# Access control
|
|
S3_ENABLE_VERSIONING=true
|
|
S3_ENABLE_MFA_DELETE=false # Enable in production for critical buckets
|
|
|
|
# =============================================================================
|
|
# Performance Settings
|
|
# =============================================================================
|
|
|
|
# S3 Upload settings
|
|
S3_MULTIPART_THRESHOLD=8388608 # 8MB threshold for multipart uploads
|
|
S3_MAX_CONCURRENT_UPLOADS=4
|
|
S3_PART_SIZE=5242880 # 5MB part size
|
|
|
|
# Retry configuration
|
|
S3_MAX_RETRIES=3
|
|
S3_RETRY_BACKOFF_MS=1000
|
|
|
|
# =============================================================================
|
|
# Model Storage Configuration
|
|
# =============================================================================
|
|
|
|
# Model storage settings
|
|
MODEL_STORAGE_TYPE=s3 # Options: local, s3
|
|
MODEL_ENABLE_COMPRESSION=true
|
|
MODEL_COMPRESSION_LEVEL=6
|
|
|
|
# Checkpoint storage
|
|
CHECKPOINT_STORAGE_TYPE=s3 # Options: local, s3
|
|
CHECKPOINT_RETENTION_COUNT=10 # Number of checkpoints to keep
|
|
|
|
# =============================================================================
|
|
# Development and Testing
|
|
# =============================================================================
|
|
|
|
# Test configuration
|
|
TEST_POSITIONS=false
|
|
TEST_S3_BUCKET=foxhunt-test-bucket
|
|
TEST_AWS_REGION=us-east-1
|
|
|
|
# Mock S3 for testing (LocalStack)
|
|
USE_LOCALSTACK=false
|
|
LOCALSTACK_ENDPOINT=http://localhost:4566
|
|
|
|
# =============================================================================
|
|
# DBN Real Data Configuration (Wave 153 - Real Data Integration)
|
|
# =============================================================================
|
|
|
|
# Enable DBN file-based market data instead of Databento API
|
|
# Set to "true" for E2E tests with real historical data
|
|
# Set to "false" or leave unset for production Databento API usage
|
|
USE_DBN_DATA=false
|
|
|
|
# DBN Symbol Mappings - Comma-separated list of symbol:path pairs
|
|
# Example: ES.FUT:/path/to/ES.FUT_ohlcv-1m_2024-01-02.dbn,NQ.FUT:/path/to/NQ.FUT.dbn
|
|
# For Docker: Use /workspace/test_data/real/databento/ prefix
|
|
# For local: Use test_data/real/databento/ (relative to workspace root)
|
|
DBN_SYMBOL_MAPPINGS=ES.FUT:test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn
|
|
|
|
# DBN Symbol Remapping - Map requested symbols to available data symbols
|
|
# Example: BTC/USD:ES.FUT,ETH/USD:ES.FUT (use ES.FUT data for crypto symbols in tests)
|
|
# This allows E2E tests requesting crypto symbols to use available futures data
|
|
DBN_SYMBOL_MAP=BTC/USD:ES.FUT,ETH/USD:ES.FUT
|
|
|
|
# =============================================================================
|
|
# Production Notes
|
|
# =============================================================================
|
|
|
|
# 1. Never commit .env files to version control
|
|
# 2. Use Vault for secure credential management in production
|
|
# 3. Enable MFA and proper IAM policies for AWS access
|
|
# 4. Monitor S3 costs and optimize storage classes
|
|
# 5. Set up proper lifecycle policies for automated cost optimization
|
|
# 6. Enable CloudTrail for audit logging
|
|
# 7. Use separate buckets for different environments (dev/staging/prod) |