refactor: restructure repo — crates/, bin/, testing/ layout
Move 17 library crates into crates/, CLI binary into bin/fxt, consolidate 10 test crates into testing/, split config crate from deployment config files. Root directory reduced from 38+ to ~17 directories. All Cargo.toml paths and build.rs proto refs updated. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
244
crates/ml/Cargo.toml
Normal file
244
crates/ml/Cargo.toml
Normal file
@@ -0,0 +1,244 @@
|
||||
[package]
|
||||
name = "ml"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
authors.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
homepage.workspace = true
|
||||
documentation.workspace = true
|
||||
publish.workspace = true
|
||||
keywords.workspace = true
|
||||
categories.workspace = true
|
||||
|
||||
[features]
|
||||
# MINIMAL features for HFT inference only - ALL HEAVY ML REMOVED
|
||||
# CUDA is now default for training - GPU acceleration mandatory
|
||||
default = ["minimal-inference", "cuda"]
|
||||
|
||||
# PRODUCTION FEATURES - LIGHTWEIGHT ONLY
|
||||
minimal-inference = [] # Minimal inference with no optional deps
|
||||
financial = [] # Basic financial calculations
|
||||
high-precision = ["rust_decimal/serde-float"]
|
||||
|
||||
# PERFORMANCE FEATURES - NO HEAVY ML
|
||||
mimalloc-allocator = ["mimalloc"] # Fast memory allocator for 10-25% speedup
|
||||
simd = [] # SIMD without heavy dependencies
|
||||
|
||||
# Storage and memory management features
|
||||
gc = [] # Garbage collection features
|
||||
s3-storage = ["aws-config", "aws-sdk-s3", "aws-types", "aws-credential-types", "urlencoding"] # S3 storage backend with AWS SDK
|
||||
cuda = ["candle-core/cuda", "candle-core/cudnn", "candle-nn/cuda", "candle-nn/cudnn"] # CUDA support (includes LSTM sigmoid ops) - OPTIONAL for CI/Docker
|
||||
|
||||
# ALL HEAVY ML FEATURES REMOVED:
|
||||
# gpu, pytorch, linfa-ml - MOVED TO ml_training_service
|
||||
# optimization, graph-models, reinforcement-learning - MOVED TO ml_training_service
|
||||
# transformers-advanced - MOVED TO ml_training_service
|
||||
|
||||
[dependencies]
|
||||
# Core async and utilities
|
||||
tokio.workspace = true
|
||||
futures.workspace = true
|
||||
async-trait.workspace = true
|
||||
clap.workspace = true # CLI argument parsing for train_tft binary
|
||||
|
||||
# Serialization and error handling
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde_yaml = "0.9" # YAML serialization for hyperparameter exports
|
||||
toml.workspace = true
|
||||
uuid.workspace = true
|
||||
thiserror.workspace = true
|
||||
anyhow.workspace = true
|
||||
chrono.workspace = true
|
||||
chrono-tz = "0.10" # Timezone support for market hours calculations (Wave C)
|
||||
csv = "1.3" # CSV serialization for action export (Wave 3 Task 3.1)
|
||||
rand.workspace = true
|
||||
rand_chacha = "0.3" # ChaCha RNG for reproducible Monte Carlo permutation tests
|
||||
|
||||
# System and I/O
|
||||
memmap2.workspace = true
|
||||
tempfile.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true # For train_tft binary logging
|
||||
prometheus.workspace = true
|
||||
reqwest.workspace = true
|
||||
colored = "2.1" # Terminal color output for evaluation reports
|
||||
|
||||
# Internal workspace crates
|
||||
trading_engine.workspace = true
|
||||
config.workspace = true
|
||||
common.workspace = true
|
||||
risk = { path = "../risk" }
|
||||
# Model loading functionality is in storage crate
|
||||
storage = { path = "../storage" }
|
||||
# Data crate for test helpers (dev-dependency in tests)
|
||||
data = { path = "../data" }
|
||||
|
||||
# Database for model registry
|
||||
sqlx.workspace = true
|
||||
|
||||
|
||||
# Essential ML frameworks for HFT inference - CUDA OPTIONAL
|
||||
# Using specific git rev (671de1db) for cudarc 0.17.3 CUDA 13.0 compatibility
|
||||
# Rev 671de1db is v0.9.1 + cudarc 0.17.3 upgrade
|
||||
# CUDA features are optional - controlled by 'cuda' feature flag
|
||||
candle-core = { git = "https://github.com/huggingface/candle", rev = "671de1db" } # Base without GPU
|
||||
candle-nn = { git = "https://github.com/huggingface/candle", rev = "671de1db" }
|
||||
# Use git version of candle-optimisers to match candle version
|
||||
candle-optimisers = { git = "https://github.com/KGrewal1/optimisers" } # Base without GPU
|
||||
|
||||
# HEAVY ML FRAMEWORKS REMOVED - MOVED TO ml_training_service
|
||||
# ort (ONNX Runtime) - REMOVED (1000+ dependencies alone!)
|
||||
# tch, torch-sys (PyTorch bindings) - REMOVED (500+ dependencies!)
|
||||
|
||||
|
||||
# Mathematical libraries
|
||||
# BLAS feature temporarily disabled - requires libopenblas-dev installation
|
||||
# TODO: Re-enable after running: sudo apt-get install -y libopenblas-dev
|
||||
ndarray = { workspace = true, features = ["rayon"] }
|
||||
nalgebra = { version = "0.33", features = ["serde-serialize"] }
|
||||
arrayfire = { version = "3.8", optional = true }
|
||||
|
||||
# MINIMAL statistics only - ALL HEAVY ML ALGORITHMS REMOVED
|
||||
# linfa ecosystem (linfa, linfa-clustering, linfa-linear, linfa-reduction) - REMOVED (200+ deps)
|
||||
# smartcore - REMOVED (100+ dependencies)
|
||||
# Basic statistics - always included (not optional)
|
||||
statrs.workspace = true # Required for statistical computations
|
||||
|
||||
|
||||
rust_decimal.workspace = true
|
||||
|
||||
|
||||
# gymnasium, rerun - REMOVED (RL frameworks moved to ml_training_service)
|
||||
|
||||
|
||||
# cudarc, wgpu - REMOVED (GPU frameworks moved to ml_training_service)
|
||||
rayon.workspace = true
|
||||
crossbeam = { version = "0.8", features = ["std"] }
|
||||
|
||||
|
||||
petgraph = { version = "0.6", features = ["serde"] } # Required for TGNN graphs
|
||||
semver = "1.0"
|
||||
lru.workspace = true # Required for model caching
|
||||
|
||||
|
||||
|
||||
# chronoutil, ta, polars - REMOVED or moved to workspace dependencies
|
||||
|
||||
|
||||
# argmin, nlopt, ipopt - REMOVED (optimization frameworks moved to ml_training_service)
|
||||
|
||||
|
||||
half = { version = "2.6.0", features = ["serde"] }
|
||||
rand_distr.workspace = true
|
||||
dbn.workspace = true # Databento Binary format for real market data loading
|
||||
databento = "0.34" # Databento API client for downloading data (includes async by default)
|
||||
|
||||
# Performance allocators
|
||||
mimalloc = { version = "0.1", optional = true } # Fast memory allocator
|
||||
dotenv = "0.15" # Load .env files for API keys
|
||||
parking_lot = { version = "0.12", features = ["hardware-lock-elision"] }
|
||||
dashmap = { version = "6.1", features = ["serde"] }
|
||||
once_cell = "1.19"
|
||||
lazy_static.workspace = true
|
||||
flate2 = "1.0"
|
||||
sha2 = "0.10"
|
||||
hmac = "0.12" # HMAC for checkpoint signatures (SEC-001 fix)
|
||||
hex = "0.4" # Hex encoding for signatures
|
||||
bincode = "1.3"
|
||||
fastrand = "2.1"
|
||||
# wide - REMOVED (SIMD moved to trading_engine)
|
||||
num-traits = "0.2"
|
||||
|
||||
# Parquet I/O for feature caching (Wave 2 Agent 8)
|
||||
# Updated to workspace version 56 to fix arrow-arith compilation conflict
|
||||
parquet.workspace = true
|
||||
arrow.workspace = true
|
||||
bytes = "1.5" # For Parquet in-memory serialization
|
||||
num = "0.4"
|
||||
libc = "0.2"
|
||||
fs2 = "0.4"
|
||||
num_cpus = "1.16"
|
||||
approx.workspace = true
|
||||
sysinfo = "0.33" # System information for benchmarks
|
||||
|
||||
# AWS SDK dependencies for S3 checkpoint storage (optional, s3-storage feature)
|
||||
aws-config = { version = "1.1", optional = true }
|
||||
aws-sdk-s3 = { version = "1.14", optional = true }
|
||||
aws-types = { version = "1.1", optional = true }
|
||||
aws-credential-types = { version = "1.1", optional = true }
|
||||
urlencoding = { version = "2.1", optional = true }
|
||||
|
||||
# Bayesian optimization for hyperparameter tuning (using argmin instead of egobox due to ndarray conflict)
|
||||
argmin = { version = "0.8", features = ["rayon"] } # Optimization framework with parallel execution
|
||||
argmin-math = "0.3" # Math utilities for argmin
|
||||
|
||||
[dev-dependencies]
|
||||
tokio-test = "0.4"
|
||||
proptest = "1.5"
|
||||
tempfile = "3.12"
|
||||
futures-test = "0.3"
|
||||
test-case = "3.0"
|
||||
rstest = "0.22"
|
||||
criterion = { version = "0.5", features = ["html_reports", "async_tokio"] }
|
||||
fastrand = "2.1"
|
||||
|
||||
tokio = { workspace = true, features = ["test-util", "macros"] }
|
||||
insta = "1.34" # Snapshot testing for ML outputs
|
||||
serial_test = "3.0" # Sequential testing for GPU resources
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
|
||||
rand_chacha = "0.3" # ChaCha RNG for hyperopt tests
|
||||
|
||||
[[example]]
|
||||
name = "cuda_test"
|
||||
path = "examples/cuda_test.rs"
|
||||
|
||||
[[example]]
|
||||
name = "gpu_training_benchmark"
|
||||
path = "examples/gpu_training_benchmark.rs"
|
||||
|
||||
[[example]]
|
||||
name = "evaluate_ppo"
|
||||
path = "examples/evaluate_ppo.rs"
|
||||
required-features = ["cuda"]
|
||||
|
||||
[[example]]
|
||||
name = "train_baseline"
|
||||
path = "examples/train_baseline.rs"
|
||||
|
||||
[[bench]]
|
||||
name = "microstructure_bench"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "alternative_bars_bench"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "wave_d_features_bench"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "bench_feature_extraction"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "tft_int8_memory_bench"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "tft_int8_inference_bench"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "tft_int8_accuracy_bench"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "hyperopt_bench"
|
||||
harness = false
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
Reference in New Issue
Block a user