e8a68ee39f
- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API - Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT) - Files saved to test_data/real/databento/ml_training/ - Total: 360 files, 15 MB compressed DBN format - Used existing Rust pattern from download_nq_fut.rs - API key loaded from .env file - 100% success rate (360/360 files) - Ready for ML training benchmarks Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
199 lines
9.1 KiB
JSON
199 lines
9.1 KiB
JSON
{
|
|
"source": "Kaggle",
|
|
"dataset_author": "imranbukhari",
|
|
"dataset_name_btc": "comprehensive-btcusd-1m-data",
|
|
"dataset_name_eth": "comprehensive-ethusd-1m-data",
|
|
"dataset_url_btc": "https://www.kaggle.com/datasets/imranbukhari/comprehensive-btcusd-1m-data",
|
|
"dataset_url_eth": "https://www.kaggle.com/datasets/imranbukhari/comprehensive-ethusd-1m-data",
|
|
"btc_usd": {
|
|
"latest_update": "2025-10-10",
|
|
"dataset_size_mb": 262,
|
|
"date_range": "Full historical data from earliest available (likely 2017-2018) to October 2025",
|
|
"update_frequency": "Daily",
|
|
"exchanges_included": 7,
|
|
"exchange_list": ["Binance", "Coinbase", "Bitfinex", "Bitstamp", "BitMEX", "KuCoin", "OKX"],
|
|
"estimated_row_count": "~3.8M rows (estimated from 262MB CSV)",
|
|
"completeness_description": "No gaps, continuous unbroken time series",
|
|
"preprocessing_applied": true,
|
|
"preprocessing_details": [
|
|
"Multi-exchange aggregation (7 exchanges)",
|
|
"Deduplication of timestamps",
|
|
"Chronological sorting",
|
|
"UTC timestamp standardization",
|
|
"Gap filling where possible",
|
|
"Outlier and anomaly filtering",
|
|
"Combined index dataset available"
|
|
],
|
|
"ohlcv_structure": {
|
|
"timestamp": "YYYY-MM-DD HH:MM:SS (UTC)",
|
|
"open": "Price at interval start",
|
|
"high": "Highest price in interval",
|
|
"low": "Lowest price in interval",
|
|
"close": "Price at interval end",
|
|
"volume": "Total traded volume (BTC)",
|
|
"optional_exchange": "Source exchange identifier"
|
|
},
|
|
"data_quality_score": 9.5,
|
|
"usability_score": 10.0,
|
|
"downloads": 3073,
|
|
"views": 11900,
|
|
"files_included": [
|
|
"BTCUSD_1m_Binance.csv",
|
|
"BTCUSD_1m_Coinbase.csv",
|
|
"BTCUSD_1m_Bitfinex.csv",
|
|
"BTCUSD_1m_Bitstamp.csv",
|
|
"BTCUSD_1m_BitMEX.csv",
|
|
"BTCUSD_1m_KuCoin.csv",
|
|
"BTCUSD_1m_OKX.csv",
|
|
"BTCUSD_1m_Combined_Index.csv"
|
|
]
|
|
},
|
|
"eth_usd": {
|
|
"latest_update": "2025-09-15",
|
|
"dataset_size_mb": 22,
|
|
"date_range": "Full historical data from earliest available (likely 2018-2019) to September 2025",
|
|
"update_frequency": "Monthly (less frequent than BTC)",
|
|
"exchanges_included": 7,
|
|
"exchange_list": ["Binance", "Coinbase", "Bitfinex", "Bitstamp", "Kraken", "KuCoin", "OKX"],
|
|
"estimated_row_count": "~320K rows (estimated from 22MB CSV)",
|
|
"completeness_description": "No gaps, continuous unbroken time series",
|
|
"preprocessing_applied": true,
|
|
"preprocessing_details": [
|
|
"Multi-exchange aggregation (7 exchanges)",
|
|
"Deduplication of timestamps",
|
|
"Chronological sorting",
|
|
"UTC timestamp standardization",
|
|
"Gap filling where possible",
|
|
"Outlier and anomaly filtering",
|
|
"Combined index dataset available"
|
|
],
|
|
"ohlcv_structure": {
|
|
"timestamp": "YYYY-MM-DD HH:MM:SS (UTC)",
|
|
"open": "Price at interval start",
|
|
"high": "Highest price in interval",
|
|
"low": "Lowest price in interval",
|
|
"close": "Price at interval end",
|
|
"volume": "Total traded volume (ETH)",
|
|
"optional_exchange": "Source exchange identifier"
|
|
},
|
|
"data_quality_score": 9.5,
|
|
"usability_score": 10.0,
|
|
"downloads": "Not specified (newer dataset)",
|
|
"views": "Not specified",
|
|
"files_included": [
|
|
"ETHUSD_1m_Binance.csv",
|
|
"ETHUSD_1m_Coinbase.csv",
|
|
"ETHUSD_1m_Bitfinex.csv",
|
|
"ETHUSD_1m_Bitstamp.csv",
|
|
"ETHUSD_1m_Kraken.csv",
|
|
"ETHUSD_1m_KuCoin.csv",
|
|
"ETHUSD_1m_OKX.csv",
|
|
"ETHUSD_1m_Combined_Index.csv"
|
|
]
|
|
},
|
|
"overall_quality_score": 9.5,
|
|
"pros": [
|
|
"FREE - No API costs or subscriptions required",
|
|
"Daily updates for BTC (4 days ago as of 2025-10-12)",
|
|
"Multi-exchange aggregation (7 major exchanges)",
|
|
"Combined index dataset - longest continuous data available on Kaggle",
|
|
"No gaps or missing data - fully continuous time series",
|
|
"Cleaned and validated - outliers and anomalies filtered",
|
|
"Standard OHLCV format - easy to parse CSV",
|
|
"High usability score (10.0) - well-documented",
|
|
"Popular dataset (3,073 downloads, 11.9K views for BTC)",
|
|
"Includes diagnostic notebooks for data quality checks",
|
|
"Better ML performance - superior MAE vs single-source datasets",
|
|
"License: CC BY-SA 4.0 (permissive, attribution required)",
|
|
"Large historical coverage - full data since exchange launch",
|
|
"UTC standardized timestamps - no timezone issues",
|
|
"Multiple file options - individual exchanges + combined index"
|
|
],
|
|
"cons": [
|
|
"Preprocessing applied - NOT raw data (aggregated, cleaned, gap-filled)",
|
|
"ETH updates less frequent than BTC (monthly vs daily)",
|
|
"Requires Kaggle account for download",
|
|
"No real-time data - 4-day lag for BTC, 1-month for ETH",
|
|
"May have minor residual gaps during extreme volatility",
|
|
"Exchange column optional - may not identify data source in all files",
|
|
"Combined index uses averaging - may smooth out exchange-specific anomalies",
|
|
"Large file sizes (262MB BTC, 22MB ETH) - storage/bandwidth considerations",
|
|
"Unknown exact start dates without downloading",
|
|
"Preprocessing methodology not fully transparent",
|
|
"Gap filling strategy unclear - interpolation vs forward-fill vs nulls",
|
|
"Volume units not explicitly specified (assumed base currency)",
|
|
"No tick-level data - only 1-minute bars"
|
|
],
|
|
"validation_concerns": [
|
|
"PREPROCESSING RED FLAG: Data is aggregated and cleaned, not raw",
|
|
"OHLCV validation needed: Verify High >= Open/Close/Low, Low <= all",
|
|
"Timestamp continuity: Check for missing 1-minute intervals",
|
|
"Volume validation: Check for zero/negative/outlier volumes",
|
|
"Price spikes: Validate against known market events",
|
|
"Exchange consistency: Compare individual files vs combined index",
|
|
"Gap analysis: Confirm claimed 'no gaps' assertion",
|
|
"Start date uncertainty: Unknown exact date range without download"
|
|
],
|
|
"download_access": {
|
|
"method": "Kaggle website or Kaggle API",
|
|
"requirements": [
|
|
"Kaggle account (free)",
|
|
"Accept dataset license (CC BY-SA 4.0)",
|
|
"Optional: Kaggle API token for programmatic access"
|
|
],
|
|
"api_command_btc": "kaggle datasets download -d imranbukhari/comprehensive-btcusd-1m-data",
|
|
"api_command_eth": "kaggle datasets download -d imranbukhari/comprehensive-ethusd-1m-data",
|
|
"sample_download_feasible": true,
|
|
"sample_download_note": "Cannot extract single day without downloading full dataset (262MB BTC, 22MB ETH)"
|
|
},
|
|
"comparison_notes": {
|
|
"vs_paid_apis": "Kaggle free but 4-day lag vs real-time paid APIs",
|
|
"vs_exchange_apis": "Pre-aggregated/cleaned vs raw from single exchange",
|
|
"vs_other_kaggle": "Most comprehensive, best usability (10.0), daily updates"
|
|
},
|
|
"recommendation": "EXCELLENT for backtesting and ML training, but NOT suitable for live trading",
|
|
"detailed_assessment": {
|
|
"data_recency": "BTC: 4 days old (2025-10-10) - GOOD for backtesting, ETH: 1 month old - ACCEPTABLE",
|
|
"data_completeness": "Claims no gaps - MUST VERIFY with actual download",
|
|
"preprocessing_transparency": "Medium - aggregation method documented, gap-filling unclear",
|
|
"production_readiness": "NOT suitable for live trading (lag + preprocessing), EXCELLENT for research/backtesting",
|
|
"ml_suitability": "EXCELLENT - author claims superior MAE performance",
|
|
"cost_effectiveness": "OUTSTANDING - completely free, high quality",
|
|
"ease_of_use": "HIGH - standard CSV, well-documented, diagnostic notebooks included",
|
|
"trust_score": "HIGH - popular (3K+ downloads), high usability (10.0), active maintenance"
|
|
},
|
|
"next_steps": [
|
|
"Download BTC dataset (262MB) using Kaggle API",
|
|
"Download ETH dataset (22MB) using Kaggle API",
|
|
"Extract 1-day sample for validation (e.g., 2024-10-01)",
|
|
"Run OHLCV validation suite (High >= Low, etc.)",
|
|
"Check timestamp continuity (should be 1440 rows per day)",
|
|
"Calculate completeness percentage",
|
|
"Compare individual exchange files vs combined index",
|
|
"Validate volume units and zero-volume percentage",
|
|
"Check for outliers and anomalies",
|
|
"Compare with other Wave 153 data sources (Binance, CoinGecko)"
|
|
],
|
|
"alternative_datasets": {
|
|
"srisahithis_btc": {
|
|
"name": "Bitcoin OHLCV Data (2019-2024) Multi-Timeframe",
|
|
"url": "https://www.kaggle.com/datasets/srisahithis/bitcoin-ohlcv-data-20192024-multi-timeframe",
|
|
"date_range": "2019-12-01 to 2024-01-01",
|
|
"status": "OUTDATED - ends Jan 2024, no recent updates"
|
|
},
|
|
"srisahithis_eth": {
|
|
"name": "Multi-Timeframe ETH/USDT OHLCV Data (2019-2024)",
|
|
"url": "https://www.kaggle.com/datasets/srisahithis/multi-timeframe-ethusdt-ohlcv-data-20192024",
|
|
"date_range": "2019-12-01 to 2024-01-01",
|
|
"status": "OUTDATED - ends Jan 2024, no recent updates"
|
|
},
|
|
"patrickgendotti": {
|
|
"name": "BTC and ETH 1-min Price History",
|
|
"url": "https://www.kaggle.com/datasets/patrickgendotti/btc-and-eth-1min-price-history",
|
|
"note": "Older dataset, date range unknown, lower usability"
|
|
}
|
|
},
|
|
"final_verdict": "BEST FREE OPTION for Wave 153 bake-off - High quality, actively maintained, comprehensive coverage",
|
|
"wave153_score": 9.5
|
|
}
|