e8a68ee39f
- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API - Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT) - Files saved to test_data/real/databento/ml_training/ - Total: 360 files, 15 MB compressed DBN format - Used existing Rust pattern from download_nq_fut.rs - API key loaded from .env file - 100% success rate (360/360 files) - Ready for ML training benchmarks Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
108 lines
4.7 KiB
JSON
108 lines
4.7 KiB
JSON
{
|
|
"source": "Kraken",
|
|
"download_method": "REST API + Google Drive Bulk Files",
|
|
"authentication_required": false,
|
|
"date_tested": "2024-10-12 (API returns only recent data)",
|
|
"api_limitations": "Critical: API only returns last 720 records regardless of 'since' parameter - NOT suitable for historical data download",
|
|
"bulk_download_method": {
|
|
"method": "Google Drive",
|
|
"url": "https://drive.google.com/file/d/1ptNqWYidLkhb2VAKuLCxmp2OXEfGO-AP/view",
|
|
"format": "ZIP containing CSV files for all pairs",
|
|
"intervals": "1, 5, 15, 30, 60, 240, 720, 1440 minute intervals",
|
|
"coverage": "From market inception to Q3 2024",
|
|
"size": "~7GB+ (all pairs, all intervals)",
|
|
"authentication": "None required",
|
|
"note": "Requires manual download (cannot be automated via API)"
|
|
},
|
|
"btc_usd": {
|
|
"pair_name": "XXBTZUSD",
|
|
"api_test_date": "2024-10-12",
|
|
"row_count": 721,
|
|
"data_range": "Last ~12 hours only (API limitation)",
|
|
"completeness_pct": 100.0,
|
|
"ohlcv_violations": 0,
|
|
"outliers_count": 0,
|
|
"zero_volume_pct": 0.0,
|
|
"price_range": "$111,679 - $111,770",
|
|
"avg_volume": "~0.05 BTC per minute",
|
|
"note": "API quality excellent but CANNOT retrieve Sept 2024 data"
|
|
},
|
|
"eth_usd": {
|
|
"pair_name": "XETHZUSD (or ETHUSD)",
|
|
"api_test_date": "2024-10-12",
|
|
"row_count": 721,
|
|
"data_range": "Last ~12 hours only (API limitation)",
|
|
"completeness_pct": 100.0,
|
|
"ohlcv_violations": 0,
|
|
"outliers_count": 0,
|
|
"zero_volume_pct": 0.0,
|
|
"price_range": "$3,826 - $3,834",
|
|
"avg_volume": "~50 ETH per minute",
|
|
"note": "API quality excellent but CANNOT retrieve Sept 2024 data"
|
|
},
|
|
"data_quality_analysis": {
|
|
"api_data_quality": 10.0,
|
|
"historical_access": 0.0,
|
|
"overall_quality_score": 5.0,
|
|
"reasoning": "API data quality is perfect (10/10), but historical data access via API is impossible (0/10). Bulk files required for historical data."
|
|
},
|
|
"pros": [
|
|
"No authentication required",
|
|
"Free historical data (via bulk download)",
|
|
"Perfect data quality (no violations, no gaps, official exchange data)",
|
|
"Includes VWAP and trade count (additional valuable metrics)",
|
|
"Multiple timeframes available (1m, 5m, 15m, 30m, 1h, 4h, 12h, 1d)",
|
|
"Data from market inception (BTC since 2013, ETH since 2016)",
|
|
"Official source - most reliable and authoritative"
|
|
],
|
|
"cons": [
|
|
"CRITICAL: API only returns last 720 records (cannot retrieve historical data programmatically)",
|
|
"Bulk files require manual download from Google Drive (cannot automate)",
|
|
"Bulk file is very large (~7GB+ for all pairs)",
|
|
"No way to download specific date ranges via API",
|
|
"Quarterly updates only (not real-time bulk files)",
|
|
"API rate limit: 1 request per second"
|
|
],
|
|
"comparison_with_cryptodatadownload": {
|
|
"kraken_advantages": [
|
|
"Official exchange data (primary source)",
|
|
"Includes VWAP and trade count",
|
|
"More timeframes available",
|
|
"Data from market inception"
|
|
],
|
|
"cryptodatadownload_advantages": [
|
|
"Easy programmatic access (direct CSV download)",
|
|
"Smaller file sizes (per-pair downloads)",
|
|
"Daily updates",
|
|
"No manual intervention required"
|
|
],
|
|
"recommendation": "Use CryptoDataDownload for automated pipelines. Use Kraken bulk files for comprehensive historical analysis."
|
|
},
|
|
"workflow_for_sept_2024_data": {
|
|
"step_1": "Manual: Download 7GB+ ZIP from Google Drive",
|
|
"step_2": "Extract: Unzip to get XXBTZUSD_1.csv and XETHZUSD_1.csv",
|
|
"step_3": "Filter: Extract Sept 1, 2024 data from full dataset",
|
|
"estimated_time": "30-60 minutes (download + extraction + filtering)",
|
|
"automation_possible": false
|
|
},
|
|
"recommendation": "NOT RECOMMENDED for automated data pipelines. While Kraken data quality is perfect (10/10), the inability to programmatically download historical data makes it unsuitable for production use. Use CryptoDataDownload or other sources that provide direct API/CSV access for specific date ranges.",
|
|
"production_suitability": {
|
|
"automated_pipelines": "Poor (1/10) - Manual download required",
|
|
"one_time_analysis": "Excellent (10/10) - Comprehensive historical data",
|
|
"data_quality": "Excellent (10/10) - Official exchange data",
|
|
"ease_of_use": "Poor (2/10) - Large files, manual process",
|
|
"overall": "Poor for automation, Excellent for manual analysis"
|
|
},
|
|
"technical_notes": {
|
|
"api_endpoint": "https://api.kraken.com/0/public/OHLC",
|
|
"pair_names": {
|
|
"btc_usd": "XXBTZUSD",
|
|
"eth_usd": "XETHZUSD or ETHUSD"
|
|
},
|
|
"ohlcv_format": "[timestamp, open, high, low, close, vwap, volume, count]",
|
|
"interval_parameter": "1 = 1 minute, 5 = 5 minutes, etc.",
|
|
"max_records_per_call": 720,
|
|
"rate_limit": "1 request per second (recommended)"
|
|
}
|
|
}
|