gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: rename test_data/mbp10 to mbp10-fixtures, remove uncompressed .dbn

Browse Source 
Removed 2.6GB of uncompressed .dbn files (derivable from .zst).
Renamed directory to mbp10-fixtures to distinguish unit test
fixtures from training data. Updated all test references to use
.dbn.zst paths directly.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-01 01:13:19 +02:00
parent a34a4634ed
commit 30c1801582
4 changed files with 26 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
crates/ml/src/feature_cache.rs
View File

@@ -17,7 +17,7 @@
//! //!
//! let cache_key = calculate_cache_key( //! let cache_key = calculate_cache_key(
//! Path::new("test_data/ES_FUT_180d.parquet"), //! Path::new("test_data/ES_FUT_180d.parquet"),
//! Path::new("test_data/mbp10"), //! Path::new("test_data/mbp10-fixtures"),
//! 50, // warmup period //! 50, // warmup period
//! )?; //! )?;
//! //!
@@ -630,7 +630,7 @@ mod tests {
fn test_cache_key_deterministic() { fn test_cache_key_deterministic() {
// Cache key should be deterministic for same inputs // Cache key should be deterministic for same inputs
let parquet = PathBuf::from("test_data/ES_FUT_180d.parquet"); let parquet = PathBuf::from("test_data/ES_FUT_180d.parquet");
let mbp10 = PathBuf::from("test_data/mbp10"); let mbp10 = PathBuf::from("test_data/mbp10-fixtures");
if parquet.exists() && mbp10.exists() { if parquet.exists() && mbp10.exists() {
let key1 = calculate_cache_key(&parquet, &mbp10, 50).unwrap(); let key1 = calculate_cache_key(&parquet, &mbp10, 50).unwrap();
@@ -643,7 +643,7 @@ mod tests {
#[test] #[test]
fn test_cache_key_varies_with_inputs() { fn test_cache_key_varies_with_inputs() {
let parquet = PathBuf::from("test_data/ES_FUT_180d.parquet"); let parquet = PathBuf::from("test_data/ES_FUT_180d.parquet");
let mbp10 = PathBuf::from("test_data/mbp10"); let mbp10 = PathBuf::from("test_data/mbp10-fixtures");
if parquet.exists() && mbp10.exists() { if parquet.exists() && mbp10.exists() {
let key1 = calculate_cache_key(&parquet, &mbp10, 50).unwrap(); let key1 = calculate_cache_key(&parquet, &mbp10, 50).unwrap();

2
crates/ml/src/trainers/dqn/data_loading.rs
View File

@@ -135,7 +135,7 @@ impl DQNTrainer {
// TRY CACHE FIRST // TRY CACHE FIRST
if let Some(cache_dir) = &self.feature_cache_dir { if let Some(cache_dir) = &self.feature_cache_dir {
let parquet_path_obj = Path::new(parquet_path); let parquet_path_obj = Path::new(parquet_path);
let mbp10 = Path::new("test_data/mbp10"); let mbp10 = Path::new("test_data/mbp10-fixtures");
info!("🔍 Checking feature cache..."); info!("🔍 Checking feature cache...");

16
crates/ml/tests/dqn_feature_cache_test.rs
View File

@@ -228,7 +228,7 @@ async fn test_cache_creation_success() -> Result<()> {
// Create cache // Create cache
let result = create_feature_cache( let result = create_feature_cache(
Path::new("test_data/ES_FUT_180d.parquet"), Path::new("test_data/ES_FUT_180d.parquet"),
Path::new("test_data/mbp10"), Path::new("test_data/mbp10-fixtures"),
&cache_dir, &cache_dir,
50, 50,
).await; ).await;
@@ -294,7 +294,7 @@ async fn test_cache_loading_matches_original() -> Result<()> {
// Create cache // Create cache
create_feature_cache( create_feature_cache(
Path::new("test_data/ES_FUT_180d.parquet"), Path::new("test_data/ES_FUT_180d.parquet"),
Path::new("test_data/mbp10"), Path::new("test_data/mbp10-fixtures"),
&cache_dir, &cache_dir,
50, 50,
) )
@@ -305,7 +305,7 @@ async fn test_cache_loading_matches_original() -> Result<()> {
let (cached_train, cached_val) = load_features_from_cache( let (cached_train, cached_val) = load_features_from_cache(
&cache_dir, &cache_dir,
Path::new("test_data/ES_FUT_180d.parquet"), Path::new("test_data/ES_FUT_180d.parquet"),
Path::new("test_data/mbp10"), Path::new("test_data/mbp10-fixtures"),
50, 50,
) )
.await .await
@@ -490,7 +490,7 @@ async fn test_hyperopt_with_cache() -> Result<()> {
// Pre-create cache // Pre-create cache
create_feature_cache( create_feature_cache(
Path::new("test_data/ES_FUT_180d.parquet"), Path::new("test_data/ES_FUT_180d.parquet"),
Path::new("test_data/mbp10"), Path::new("test_data/mbp10-fixtures"),
&cache_dir, &cache_dir,
50, 50,
) )
@@ -509,7 +509,7 @@ async fn test_hyperopt_with_cache() -> Result<()> {
let (_train, _val) = load_features_from_cache( let (_train, _val) = load_features_from_cache(
&cache_dir, &cache_dir,
Path::new("test_data/ES_FUT_180d.parquet"), Path::new("test_data/ES_FUT_180d.parquet"),
Path::new("test_data/mbp10"), Path::new("test_data/mbp10-fixtures"),
50, 50,
) )
.await .await
@@ -566,7 +566,7 @@ async fn test_cache_performance_benchmark() -> Result<()> {
let start = Instant::now(); let start = Instant::now();
create_feature_cache( create_feature_cache(
Path::new("test_data/ES_FUT_180d.parquet"), Path::new("test_data/ES_FUT_180d.parquet"),
Path::new("test_data/mbp10"), Path::new("test_data/mbp10-fixtures"),
&cache_dir, &cache_dir,
50, 50,
) )
@@ -590,7 +590,7 @@ async fn test_cache_performance_benchmark() -> Result<()> {
let (_train, _val) = load_features_from_cache( let (_train, _val) = load_features_from_cache(
&cache_dir, &cache_dir,
Path::new("test_data/ES_FUT_180d.parquet"), Path::new("test_data/ES_FUT_180d.parquet"),
Path::new("test_data/mbp10"), Path::new("test_data/mbp10-fixtures"),
50, 50,
) )
.await .await
@@ -643,7 +643,7 @@ async fn test_missing_cache_graceful_fallback() -> Result<()> {
let result = load_features_from_cache( let result = load_features_from_cache(
&nonexistent_cache, &nonexistent_cache,
Path::new("test_data/ES_FUT_180d.parquet"), Path::new("test_data/ES_FUT_180d.parquet"),
Path::new("test_data/mbp10"), Path::new("test_data/mbp10-fixtures"),
50, 50,
) )
.await; .await;

28
crates/ml/tests/mbp10_parsing_test.rs
View File

@@ -84,10 +84,10 @@ use tracing::{info, warn};
async fn test_parse_mbp10_single_day() { async fn test_parse_mbp10_single_day() {
// Test parsing a single day of MBP-10 data // Test parsing a single day of MBP-10 data
let parser = DbnParser::new().expect("Failed to create parser"); let parser = DbnParser::new().expect("Failed to create parser");
let path = Path::new("../test_data/mbp10/glbx-mdp3-20240102.mbp-10.dbn"); let path = Path::new("../test_data/mbp10-fixtures/glbx-mdp3-20240102.mbp-10.dbn.zst");
if !path.exists() { if !path.exists() {
warn!(path = ?path, "Test file not found, run: cd test_data/mbp10 && zstd -d *.zst"); warn!(path = ?path, "Test file not found, MBP-10 test data missing");
panic!("Test data not available"); panic!("Test data not available");
} }
@@ -177,9 +177,9 @@ async fn test_parse_mbp10_multiple_days() {
let parser = DbnParser::new().expect("Failed to create parser"); let parser = DbnParser::new().expect("Failed to create parser");
let files = vec![ let files = vec![
"../test_data/mbp10/glbx-mdp3-20240102.mbp-10.dbn", "../test_data/mbp10-fixtures/glbx-mdp3-20240102.mbp-10.dbn.zst",
"../test_data/mbp10/glbx-mdp3-20240103.mbp-10.dbn", "../test_data/mbp10-fixtures/glbx-mdp3-20240103.mbp-10.dbn.zst",
"../test_data/mbp10/glbx-mdp3-20240104.mbp-10.dbn", "../test_data/mbp10-fixtures/glbx-mdp3-20240104.mbp-10.dbn.zst",
]; ];
let mut total_snapshots = 0; let mut total_snapshots = 0;
@@ -222,7 +222,7 @@ async fn test_mbp10_parsing_performance() {
use std::time::Instant; use std::time::Instant;
let parser = DbnParser::new().expect("Failed to create parser"); let parser = DbnParser::new().expect("Failed to create parser");
let path = Path::new("../test_data/mbp10/glbx-mdp3-20240102.mbp-10.dbn"); let path = Path::new("../test_data/mbp10-fixtures/glbx-mdp3-20240102.mbp-10.dbn.zst");
if !path.exists() { if !path.exists() {
warn!("Test file not found, skipping performance test"); warn!("Test file not found, skipping performance test");
@@ -262,7 +262,7 @@ async fn test_mbp10_parsing_performance() {
async fn test_mbp10_data_quality() { async fn test_mbp10_data_quality() {
// Deep dive into data quality metrics // Deep dive into data quality metrics
let parser = DbnParser::new().expect("Failed to create parser"); let parser = DbnParser::new().expect("Failed to create parser");
let path = Path::new("../test_data/mbp10/glbx-mdp3-20240102.mbp-10.dbn"); let path = Path::new("../test_data/mbp10-fixtures/glbx-mdp3-20240102.mbp-10.dbn.zst");
if !path.exists() { if !path.exists() {
warn!("Test file not found, skipping data quality test"); warn!("Test file not found, skipping data quality test");
@@ -320,13 +320,13 @@ async fn test_mbp10_all_files_summary() {
let parser = DbnParser::new().expect("Failed to create parser"); let parser = DbnParser::new().expect("Failed to create parser");
let files = vec![ let files = vec![
"../test_data/mbp10/glbx-mdp3-20240102.mbp-10.dbn", "../test_data/mbp10-fixtures/glbx-mdp3-20240102.mbp-10.dbn.zst",
"../test_data/mbp10/glbx-mdp3-20240103.mbp-10.dbn", "../test_data/mbp10-fixtures/glbx-mdp3-20240103.mbp-10.dbn.zst",
"../test_data/mbp10/glbx-mdp3-20240104.mbp-10.dbn", "../test_data/mbp10-fixtures/glbx-mdp3-20240104.mbp-10.dbn.zst",
"../test_data/mbp10/glbx-mdp3-20240105.mbp-10.dbn", "../test_data/mbp10-fixtures/glbx-mdp3-20240105.mbp-10.dbn.zst",
"../test_data/mbp10/glbx-mdp3-20240107.mbp-10.dbn", "../test_data/mbp10-fixtures/glbx-mdp3-20240107.mbp-10.dbn.zst",
"../test_data/mbp10/glbx-mdp3-20240108.mbp-10.dbn", "../test_data/mbp10-fixtures/glbx-mdp3-20240108.mbp-10.dbn.zst",
"../test_data/mbp10/glbx-mdp3-20240109.mbp-10.dbn", "../test_data/mbp10-fixtures/glbx-mdp3-20240109.mbp-10.dbn.zst",
]; ];
info!("MBP-10 Data Summary Report"); info!("MBP-10 Data Summary Report");