gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Download 360 DBN files (36.3 MB) using Rust databento client

Browse Source 
- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API
- Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT)
- Files saved to test_data/real/databento/ml_training/
- Total: 360 files, 15 MB compressed DBN format
- Used existing Rust pattern from download_nq_fut.rs
- API key loaded from .env file
- 100% success rate (360/360 files)
- Ready for ML training benchmarks

Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
This commit is contained in:
jgrusewski
2025-10-13 13:30:02 +02:00
parent 08821565d6
commit e8a68ee39f
539 changed files with 1070448 additions and 489 deletions
Download Patch File Download Diff File Expand all files Collapse all files

340
Cargo.lock generated
View File

@@ -13,6 +13,7 @@ dependencies = [
"common",
"config",
"criterion",
"data",
"futures",
"num-traits",
"rand 0.8.5",
@@ -156,6 +157,15 @@ version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "ansi_term"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
dependencies = [
"winapi",
]
[[package]]
name = "anstream"
version = "0.6.21"
@@ -224,7 +234,7 @@ dependencies = [
"base64 0.22.1",
"bytes",
"chrono",
"clap",
"clap 4.5.48",
"common",
"config",
"criterion",
@@ -287,7 +297,7 @@ dependencies = [
"anyhow",
"base64 0.22.1",
"chrono",
"clap",
"clap 4.5.48",
"common",
"dashmap 6.1.0",
"futures",
@@ -903,6 +913,17 @@ version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "atty"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi 0.1.19",
"libc",
"winapi",
]
[[package]]
name = "autocfg"
version = "1.5.0"
@@ -1517,12 +1538,16 @@ dependencies = [
"axum 0.7.9",
"base64 0.22.1",
"chrono",
"clap 4.5.48",
"common",
"config",
"criterion",
"crossbeam",
"dashmap 6.1.0",
"data",
"dbn 0.42.0",
"dotenvy",
"futures",
"influxdb2",
"jsonwebtoken",
"ml",
@@ -2086,6 +2111,21 @@ dependencies = [
"libloading",
]
[[package]]
name = "clap"
version = "2.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
dependencies = [
"ansi_term",
"atty",
"bitflags 1.3.2",
"strsim 0.8.0",
"textwrap",
"unicode-width 0.1.14",
"vec_map",
]
[[package]]
name = "clap"
version = "4.5.48"
@@ -2262,6 +2302,8 @@ dependencies = [
"compression-core",
"flate2",
"memchr",
"zstd",
"zstd-safe",
]
[[package]]
@@ -2466,7 +2508,7 @@ dependencies = [
"anes",
"cast",
"ciborium",
"clap",
"clap 4.5.48",
"criterion-plot",
"futures",
"is-terminal",
@@ -2808,6 +2850,7 @@ dependencies = [
"crossbeam",
"crossbeam-channel",
"dashmap 6.1.0",
"dbn 0.42.0",
"fastrand",
"flate2",
"futures",
@@ -2884,6 +2927,147 @@ dependencies = [
"uuid",
]
[[package]]
name = "databento"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e97bfcbcdd210697ec11899a5e79898c4bd6e12706005d975b9c92e4e03e99c"
dependencies = [
"dbn 0.25.0",
"futures",
"hex",
"reqwest 0.12.23",
"serde",
"serde_json",
"sha2",
"thiserror 2.0.17",
"time",
"tokio",
"tokio-util",
"tracing",
"typed-builder",
]
[[package]]
name = "dbn"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8aa7989e82a5f401c984beec175ef64e451fd815c13762f4a94a8d58ddb2400a"
dependencies = [
"csv",
"dbn-macros 0.22.1",
"fallible-streaming-iterator",
"itoa",
"json-writer",
"num_enum",
"thiserror 1.0.69",
"time",
"zstd",
]
[[package]]
name = "dbn"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c960e0f7fd591cc01264124ed3462c366207e406972b111b1d3174c79946d62"
dependencies = [
"csv",
"dbn-macros 0.23.1",
"fallible-streaming-iterator",
"itoa",
"json-writer",
"num_enum",
"thiserror 2.0.17",
"time",
"zstd",
]
[[package]]
name = "dbn"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a0d47473bf3d0064cc16e7c454cc8f2802b688c7ad073b9ef9fee07deb77073"
dependencies = [
"async-compression",
"csv",
"dbn-macros 0.25.0",
"fallible-streaming-iterator",
"itoa",
"json-writer",
"num_enum",
"serde",
"thiserror 2.0.17",
"time",
"tokio",
"zstd",
]
[[package]]
name = "dbn"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fae2ff443e1ada6c0e3697ac904b8a0f236cdc33552ae494a63a74584d1a4ebe"
dependencies = [
"csv",
"dbn-macros 0.42.0",
"fallible-streaming-iterator",
"itoa",
"json-writer",
"num_enum",
"oval",
"thiserror 2.0.17",
"time",
"zstd",
]
[[package]]
name = "dbn-macros"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32f915c673fd5d1fa8a9ca66512cbcd789667bb7655eaebf9699ca1b6c8334de"
dependencies = [
"proc-macro-crate",
"proc-macro2",
"quote",
"syn 2.0.106",
]
[[package]]
name = "dbn-macros"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb5edb3c0414a5c178ae67de963544f48a5badb1a1708af4f89bc33241a9eae6"
dependencies = [
"proc-macro-crate",
"proc-macro2",
"quote",
"syn 2.0.106",
]
[[package]]
name = "dbn-macros"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c7dc21ecca9d911faf9536cf4b735210fa22d0d1f6e7b7b581a02f84761d6d9"
dependencies = [
"proc-macro-crate",
"proc-macro2",
"quote",
"syn 2.0.106",
]
[[package]]
name = "dbn-macros"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f04599e0929d4fd660d6918a059706057bf4c2626273896d4c862fb18239860a"
dependencies = [
"proc-macro-crate",
"proc-macro2",
"quote",
"syn 2.0.106",
]
[[package]]
name = "deadpool"
version = "0.12.3"
@@ -3064,6 +3248,12 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
[[package]]
name = "dotenv"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
[[package]]
name = "dotenvy"
version = "0.15.7"
@@ -3267,6 +3457,12 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "fastrand"
version = "2.3.0"
@@ -3530,7 +3726,7 @@ dependencies = [
"assert_matches",
"bigdecimal",
"chrono",
"clap",
"clap 4.5.48",
"common",
"config",
"criterion",
@@ -4103,6 +4299,15 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hermit-abi"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
dependencies = [
"libc",
]
[[package]]
name = "hermit-abi"
version = "0.5.2"
@@ -4719,13 +4924,16 @@ name = "integration_tests"
version = "1.0.0"
dependencies = [
"anyhow",
"backtesting_service",
"chrono",
"ctor",
"dbn 0.22.1",
"dotenvy",
"futures",
"jsonwebtoken",
"prost 0.14.1",
"reqwest 0.12.23",
"rust_decimal",
"serde",
"serde_json",
"serial_test",
@@ -4782,7 +4990,7 @@ version = "0.4.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9"
dependencies = [
"hermit-abi",
"hermit-abi 0.5.2",
"libc",
"windows-sys 0.59.0",
]
@@ -4878,6 +5086,16 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "json-writer"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "279046e6427c19c86f93df06fe9dc90c32b43f4a2a85bb3083d579e4a1e7ef03"
dependencies = [
"itoa",
"ryu",
]
[[package]]
name = "jsonwebtoken"
version = "9.3.1"
@@ -5351,6 +5569,10 @@ dependencies = [
"criterion",
"crossbeam",
"dashmap 6.1.0",
"data",
"databento",
"dbn 0.23.1",
"dotenv",
"fastrand",
"flate2",
"fs2",
@@ -5387,6 +5609,7 @@ dependencies = [
"sha2",
"statrs",
"storage",
"structopt",
"sysinfo 0.33.1",
"tempfile",
"test-case",
@@ -5439,10 +5662,11 @@ dependencies = [
"bytes",
"chacha20poly1305",
"chrono",
"clap",
"clap 4.5.48",
"common",
"config",
"data",
"dbn 0.42.0",
"flate2",
"futures",
"jsonwebtoken",
@@ -5854,7 +6078,7 @@ version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
dependencies = [
"hermit-abi",
"hermit-abi 0.5.2",
"libc",
]
@@ -6035,6 +6259,12 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
[[package]]
name = "oval"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "135cef32720c6746450d910890b0b69bcba2bbf6f85c9f4583df13fe415de828"
[[package]]
name = "p256"
version = "0.11.1"
@@ -6502,6 +6732,30 @@ dependencies = [
"toml_edit 0.23.6",
]
[[package]]
name = "proc-macro-error"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
dependencies = [
"proc-macro-error-attr",
"proc-macro2",
"quote",
"syn 1.0.109",
"version_check",
]
[[package]]
name = "proc-macro-error-attr"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
dependencies = [
"proc-macro2",
"quote",
"version_check",
]
[[package]]
name = "proc-macro2"
version = "1.0.101"
@@ -8675,6 +8929,12 @@ dependencies = [
"unicode-properties",
]
[[package]]
name = "strsim"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
[[package]]
name = "strsim"
version = "0.10.0"
@@ -8687,6 +8947,30 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "structopt"
version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10"
dependencies = [
"clap 2.34.0",
"lazy_static",
"structopt-derive",
]
[[package]]
name = "structopt-derive"
version = "0.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
dependencies = [
"heck 0.3.3",
"proc-macro-error",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "strum"
version = "0.26.3"
@@ -8953,7 +9237,7 @@ dependencies = [
"arc-swap",
"async-trait",
"chrono",
"clap",
"clap 4.5.48",
"common",
"config",
"criterion",
@@ -9001,6 +9285,15 @@ dependencies = [
"uuid",
]
[[package]]
name = "textwrap"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
dependencies = [
"unicode-width 0.1.14",
]
[[package]]
name = "thiserror"
version = "1.0.69"
@@ -9774,12 +10067,13 @@ dependencies = [
"base64 0.22.1",
"bytes",
"chrono",
"clap",
"clap 4.5.48",
"common",
"config",
"criterion",
"data",
"database",
"dbn 0.23.1",
"fastrand",
"futures",
"futures-util",
@@ -9841,7 +10135,7 @@ dependencies = [
"anyhow",
"base64 0.22.1",
"chrono",
"clap",
"clap 4.5.48",
"common",
"dashmap 6.1.0",
"futures",
@@ -9927,6 +10221,26 @@ version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
[[package]]
name = "typed-builder"
version = "0.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7"
dependencies = [
"typed-builder-macro",
]
[[package]]
name = "typed-builder-macro"
version = "0.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
]
[[package]]
name = "typenum"
version = "1.19.0"
@@ -10155,6 +10469,12 @@ version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "vec_map"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
[[package]]
name = "version-compare"
version = "0.2.0"