🔧 Remove AWS SDK dependencies, partial vault fix
- Removed aws-sdk-s3 and aws-config from trading_service - Started fixing vault feature gating in config crate - Need to complete vault integration (not optional) - Need shared model loading library for trading and backtesting services
This commit is contained in:
jgrusewski
735
Cargo.lock
generated
735
Cargo.lock
generated
@@ -1181,418 +1181,6 @@ version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
|
||||
|
||||
[[package]]
|
||||
name = "aws-config"
|
||||
version = "1.5.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "90aff65e86db5fe300752551c1b015ef72b708ac54bded8ef43d0d53cb7cb0b1"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
"aws-sdk-sso",
|
||||
"aws-sdk-ssooidc",
|
||||
"aws-sdk-sts",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http 0.61.1",
|
||||
"aws-smithy-json",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"fastrand 2.3.0",
|
||||
"hex",
|
||||
"http 0.2.12",
|
||||
"ring",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"url",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-credential-types"
|
||||
version = "1.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d025db5d9f52cbc413b167136afb3d8aeea708c0d8884783cf6253be5e22f6f2"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-runtime"
|
||||
version = "1.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "76dd04d39cc12844c0994f2c9c5a6f5184c22e9188ec1ff723de41910a21dcad"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-sigv4",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-http 0.60.12",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"fastrand 2.3.0",
|
||||
"http 0.2.12",
|
||||
"http-body 0.4.6",
|
||||
"once_cell",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"tracing",
|
||||
"uuid 1.16.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.78.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3038614b6cf7dd68d9a7b5b39563d04337eb3678d1d4173e356e927b0356158a"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
"aws-sigv4",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-checksums",
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-http 0.61.1",
|
||||
"aws-smithy-json",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"aws-smithy-xml",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"fastrand 2.3.0",
|
||||
"hex",
|
||||
"hmac",
|
||||
"http 0.2.12",
|
||||
"http-body 0.4.6",
|
||||
"lru",
|
||||
"once_cell",
|
||||
"percent-encoding",
|
||||
"regex-lite",
|
||||
"sha2",
|
||||
"tracing",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sso"
|
||||
version = "1.61.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e65ff295979977039a25f5a0bf067a64bc5e6aa38f3cef4037cf42516265553c"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http 0.61.1",
|
||||
"aws-smithy-json",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"http 0.2.12",
|
||||
"once_cell",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-ssooidc"
|
||||
version = "1.62.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91430a60f754f235688387b75ee798ef00cfd09709a582be2b7525ebb5306d4f"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http 0.61.1",
|
||||
"aws-smithy-json",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"http 0.2.12",
|
||||
"once_cell",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sts"
|
||||
version = "1.62.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9276e139d39fff5a0b0c984fc2d30f970f9a202da67234f948fda02e5bea1dbe"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http 0.61.1",
|
||||
"aws-smithy-json",
|
||||
"aws-smithy-query",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"aws-smithy-xml",
|
||||
"aws-types",
|
||||
"http 0.2.12",
|
||||
"once_cell",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sigv4"
|
||||
version = "1.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "084c34162187d39e3740cb635acd73c4e3a551a36146ad6fe8883c929c9f876c"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-http 0.62.3",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"crypto-bigint 0.5.5",
|
||||
"form_urlencoded",
|
||||
"hex",
|
||||
"hmac",
|
||||
"http 0.2.12",
|
||||
"http 1.3.1",
|
||||
"p256",
|
||||
"percent-encoding",
|
||||
"ring",
|
||||
"sha2",
|
||||
"subtle",
|
||||
"time",
|
||||
"tracing",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-async"
|
||||
version = "1.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-checksums"
|
||||
version = "0.63.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56d2df0314b8e307995a3b86d44565dfe9de41f876901a7d71886c756a25979f"
|
||||
dependencies = [
|
||||
"aws-smithy-http 0.62.3",
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"crc-fast",
|
||||
"hex",
|
||||
"http 0.2.12",
|
||||
"http-body 0.4.6",
|
||||
"md-5",
|
||||
"pin-project-lite",
|
||||
"sha1",
|
||||
"sha2",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-eventstream"
|
||||
version = "0.60.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "182b03393e8c677347fb5705a04a9392695d47d20ef0a2f8cfe28c8e6b9b9778"
|
||||
dependencies = [
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"crc32fast",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http"
|
||||
version = "0.60.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"bytes-utils",
|
||||
"futures-core",
|
||||
"http 0.2.12",
|
||||
"http-body 0.4.6",
|
||||
"once_cell",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http"
|
||||
version = "0.61.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6f276f21c7921fe902826618d1423ae5bf74cf8c1b8472aee8434f3dfd31824"
|
||||
dependencies = [
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"bytes-utils",
|
||||
"futures-core",
|
||||
"http 0.2.12",
|
||||
"http-body 0.4.6",
|
||||
"once_cell",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http"
|
||||
version = "0.62.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c4dacf2d38996cf729f55e7a762b30918229917eca115de45dfa8dfb97796c9"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"bytes-utils",
|
||||
"futures-core",
|
||||
"http 0.2.12",
|
||||
"http 1.3.1",
|
||||
"http-body 0.4.6",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-json"
|
||||
version = "0.61.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eaa31b350998e703e9826b2104dd6f63be0508666e1aba88137af060e8944047"
|
||||
dependencies = [
|
||||
"aws-smithy-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-query"
|
||||
version = "0.60.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb"
|
||||
dependencies = [
|
||||
"aws-smithy-types",
|
||||
"urlencoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime"
|
||||
version = "1.7.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d526a12d9ed61fadefda24abe2e682892ba288c2018bcb38b1b4c111d13f6d92"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http 0.60.12",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"fastrand 2.3.0",
|
||||
"h2 0.3.27",
|
||||
"http 0.2.12",
|
||||
"http-body 0.4.6",
|
||||
"http-body 1.0.1",
|
||||
"httparse",
|
||||
"hyper 0.14.32",
|
||||
"hyper-rustls 0.24.2",
|
||||
"once_cell",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"rustls 0.21.12",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07f5e0fc8a6b3f2303f331b94504bbf754d85488f402d6f1dd7a6080f99afe56"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"http 0.2.12",
|
||||
"http 1.3.1",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d498595448e43de7f4296b7b7a18a8a02c61ec9349128c80a368f7c3b4ab11a8"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
"bytes-utils",
|
||||
"futures-core",
|
||||
"http 0.2.12",
|
||||
"http 1.3.1",
|
||||
"http-body 0.4.6",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"itoa",
|
||||
"num-integer",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"ryu",
|
||||
"serde",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-xml"
|
||||
version = "0.60.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3db87b96cb1b16c024980f133968d52882ca0daaee3a086c6decc500f6c99728"
|
||||
dependencies = [
|
||||
"xmlparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-types"
|
||||
version = "1.3.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b069d19bf01e46298eaedd7c6f283fe565a59263e53eebec945f3e6398f42390"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"rustc_version 0.4.1",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.6.20"
|
||||
@@ -1801,12 +1389,6 @@ dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "base16ct"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.13.1"
|
||||
@@ -1825,16 +1407,6 @@ version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "base64-simd"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195"
|
||||
dependencies = [
|
||||
"outref",
|
||||
"vsimd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "base64ct"
|
||||
version = "1.8.0"
|
||||
@@ -2216,16 +1788,6 @@ version = "1.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
|
||||
|
||||
[[package]]
|
||||
name = "bytes-utils"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bzip2"
|
||||
version = "0.4.4"
|
||||
@@ -3078,19 +2640,6 @@ version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
|
||||
|
||||
[[package]]
|
||||
name = "crc-fast"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6bf62af4cc77d8fe1c22dde4e721d87f2f54056139d8c412e1366b740305f56f"
|
||||
dependencies = [
|
||||
"crc",
|
||||
"digest",
|
||||
"libc",
|
||||
"rand 0.9.2",
|
||||
"regex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.0"
|
||||
@@ -3255,28 +2804,6 @@ version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-bigint"
|
||||
version = "0.4.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-bigint"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
||||
dependencies = [
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.6"
|
||||
@@ -3650,16 +3177,6 @@ version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b"
|
||||
|
||||
[[package]]
|
||||
name = "der"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de"
|
||||
dependencies = [
|
||||
"const-oid",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "der"
|
||||
version = "0.7.10"
|
||||
@@ -3966,18 +3483,6 @@ dependencies = [
|
||||
"uuid 1.16.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ecdsa"
|
||||
version = "0.14.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c"
|
||||
dependencies = [
|
||||
"der 0.6.1",
|
||||
"elliptic-curve",
|
||||
"rfc6979",
|
||||
"signature 1.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ecolor"
|
||||
version = "0.28.1"
|
||||
@@ -4192,26 +3697,6 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "elliptic-curve"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"crypto-bigint 0.4.9",
|
||||
"der 0.6.1",
|
||||
"digest",
|
||||
"ff",
|
||||
"generic-array",
|
||||
"group",
|
||||
"pkcs8 0.9.0",
|
||||
"rand_core 0.6.4",
|
||||
"sec1",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "emath"
|
||||
version = "0.28.1"
|
||||
@@ -4650,16 +4135,6 @@ dependencies = [
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ff"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160"
|
||||
dependencies = [
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "filetime"
|
||||
version = "0.2.26"
|
||||
@@ -5505,17 +4980,6 @@ dependencies = [
|
||||
"bitflags 2.9.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "group"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7"
|
||||
dependencies = [
|
||||
"ff",
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gymnasium"
|
||||
version = "0.0.1"
|
||||
@@ -5985,9 +5449,7 @@ dependencies = [
|
||||
"futures-util",
|
||||
"http 0.2.12",
|
||||
"hyper 0.14.32",
|
||||
"log",
|
||||
"rustls 0.21.12",
|
||||
"rustls-native-certs 0.6.3",
|
||||
"tokio",
|
||||
"tokio-rustls 0.24.1",
|
||||
]
|
||||
@@ -6377,7 +5839,7 @@ dependencies = [
|
||||
"secrecy",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"snafu",
|
||||
"snafu 0.6.10",
|
||||
"url",
|
||||
]
|
||||
|
||||
@@ -8400,6 +7862,36 @@ dependencies = [
|
||||
"memchr 2.7.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object_store"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
"humantime 2.3.0",
|
||||
"hyper 1.7.0",
|
||||
"itertools 0.13.0",
|
||||
"md-5",
|
||||
"parking_lot 0.12.4",
|
||||
"percent-encoding",
|
||||
"quick-xml 0.36.2",
|
||||
"rand 0.8.5",
|
||||
"reqwest 0.12.12",
|
||||
"ring",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"snafu 0.7.5",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"url",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.3"
|
||||
@@ -8640,12 +8132,6 @@ dependencies = [
|
||||
"zip 0.6.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "outref"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
|
||||
|
||||
[[package]]
|
||||
name = "oval"
|
||||
version = "2.0.0"
|
||||
@@ -8667,17 +8153,6 @@ version = "4.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "48dd4f4a2c8405440fd0462561f0e5806bd0f77e86f51c761481bdd4018b545e"
|
||||
|
||||
[[package]]
|
||||
name = "p256"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594"
|
||||
dependencies = [
|
||||
"ecdsa",
|
||||
"elliptic-curve",
|
||||
"sha2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking"
|
||||
version = "2.2.1"
|
||||
@@ -9014,19 +8489,9 @@ version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f"
|
||||
dependencies = [
|
||||
"der 0.7.10",
|
||||
"pkcs8 0.10.2",
|
||||
"spki 0.7.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkcs8"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba"
|
||||
dependencies = [
|
||||
"der 0.6.1",
|
||||
"spki 0.6.0",
|
||||
"der",
|
||||
"pkcs8",
|
||||
"spki",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9035,8 +8500,8 @@ version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
|
||||
dependencies = [
|
||||
"der 0.7.10",
|
||||
"spki 0.7.3",
|
||||
"der",
|
||||
"spki",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9970,6 +9435,16 @@ version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.36.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe"
|
||||
dependencies = [
|
||||
"memchr 2.7.5",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.37.5"
|
||||
@@ -11574,12 +11049,6 @@ dependencies = [
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-lite"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "943f41321c63ef1c92fd763bfe054d2668f7f225a5c29f0105903dc2fc04ba30"
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.6"
|
||||
@@ -11746,17 +11215,6 @@ version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b3789b30bd25ba102de4beabd95d21ac45b69b1be7d14522bab988c526d6799"
|
||||
|
||||
[[package]]
|
||||
name = "rfc6979"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb"
|
||||
dependencies = [
|
||||
"crypto-bigint 0.4.9",
|
||||
"hmac",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rfd"
|
||||
version = "0.12.1"
|
||||
@@ -11947,10 +11405,10 @@ dependencies = [
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
"pkcs1",
|
||||
"pkcs8 0.10.2",
|
||||
"pkcs8",
|
||||
"rand_core 0.6.4",
|
||||
"signature 2.2.0",
|
||||
"spki 0.7.3",
|
||||
"signature",
|
||||
"spki",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
@@ -12198,18 +11656,6 @@ dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-native-certs"
|
||||
version = "0.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00"
|
||||
dependencies = [
|
||||
"openssl-probe",
|
||||
"rustls-pemfile 1.0.4",
|
||||
"schannel",
|
||||
"security-framework 2.11.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-native-certs"
|
||||
version = "0.7.3"
|
||||
@@ -12445,20 +11891,6 @@ dependencies = [
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sec1"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"der 0.6.1",
|
||||
"generic-array",
|
||||
"pkcs8 0.9.0",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "secrecy"
|
||||
version = "0.8.0"
|
||||
@@ -12816,16 +12248,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signature"
|
||||
version = "1.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c"
|
||||
dependencies = [
|
||||
"digest",
|
||||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signature"
|
||||
version = "2.2.0"
|
||||
@@ -13094,7 +12516,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eab12d3c261b2308b0d80c26fffb58d17eba81a4be97890101f416b478c79ca7"
|
||||
dependencies = [
|
||||
"doc-comment",
|
||||
"snafu-derive",
|
||||
"snafu-derive 0.6.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "snafu"
|
||||
version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6"
|
||||
dependencies = [
|
||||
"doc-comment",
|
||||
"snafu-derive 0.7.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -13108,6 +12540,18 @@ dependencies = [
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "snafu-derive"
|
||||
version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf"
|
||||
dependencies = [
|
||||
"heck 0.4.1",
|
||||
"proc-macro2 1.0.101",
|
||||
"quote 1.0.40",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "snap"
|
||||
version = "1.1.1"
|
||||
@@ -13168,16 +12612,6 @@ dependencies = [
|
||||
"bitflags 2.9.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spki"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"der 0.6.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spki"
|
||||
version = "0.7.3"
|
||||
@@ -13185,7 +12619,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"der 0.7.10",
|
||||
"der",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -13447,9 +12881,6 @@ version = "1.0.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"aws-config",
|
||||
"aws-sdk-s3",
|
||||
"aws-types",
|
||||
"backoff",
|
||||
"bincode",
|
||||
"chrono",
|
||||
@@ -13460,6 +12891,7 @@ dependencies = [
|
||||
"futures",
|
||||
"indexmap 2.11.4",
|
||||
"lru",
|
||||
"object_store",
|
||||
"parking_lot 0.12.4",
|
||||
"rustc-hash 1.1.0",
|
||||
"serde",
|
||||
@@ -14800,14 +14232,18 @@ dependencies = [
|
||||
"futures",
|
||||
"hdrhistogram",
|
||||
"hyper 1.7.0",
|
||||
"memmap2 0.9.8",
|
||||
"ml",
|
||||
"once_cell",
|
||||
"prost 0.13.5",
|
||||
"prost-build",
|
||||
"risk",
|
||||
"semver 1.0.27",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"storage",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tonic 0.12.3",
|
||||
@@ -14820,6 +14256,7 @@ dependencies = [
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"trading_engine",
|
||||
"uuid 1.16.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -15197,12 +14634,6 @@ version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "vsimd"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
|
||||
|
||||
[[package]]
|
||||
name = "vswhom"
|
||||
version = "0.1.0"
|
||||
@@ -15484,7 +14915,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "54cb1e9dc49da91950bdfd8b848c49330536d9d1fb03d4bfec8cae50caa50ae3"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.101",
|
||||
"quick-xml",
|
||||
"quick-xml 0.37.5",
|
||||
"quote 1.0.40",
|
||||
]
|
||||
|
||||
@@ -16502,12 +15933,6 @@ version = "0.8.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6fd8403733700263c6eb89f192880191f1b83e332f7a20371ddcf421c4a337c7"
|
||||
|
||||
[[package]]
|
||||
name = "xmlparser"
|
||||
version = "0.13.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
|
||||
|
||||
[[package]]
|
||||
name = "xshell"
|
||||
version = "0.2.7"
|
||||
|
||||
@@ -38,8 +38,8 @@ sqlx = { workspace = true, features = ["postgres", "runtime-tokio-rustls", "chro
|
||||
# Logging and tracing
|
||||
tracing.workspace = true
|
||||
|
||||
# HashiCorp Vault integration
|
||||
vaultrs = "0.7"
|
||||
# HashiCorp Vault integration (optional)
|
||||
vaultrs = { version = "0.7", optional = true }
|
||||
|
||||
# High-performance data structures
|
||||
rustc-hash.workspace = true
|
||||
@@ -64,7 +64,7 @@ wiremock.workspace = true
|
||||
testcontainers.workspace = true
|
||||
|
||||
[features]
|
||||
default = ["postgres", "vault"]
|
||||
default = ["postgres"]
|
||||
postgres = []
|
||||
vault = []
|
||||
vault = ["vaultrs"]
|
||||
redis = []
|
||||
@@ -80,6 +80,7 @@ pub enum ConfigError {
|
||||
source: sqlx::Error,
|
||||
},
|
||||
|
||||
#[cfg(feature = "vault")]
|
||||
#[error("Vault client error: {source}")]
|
||||
VaultClientError {
|
||||
#[from]
|
||||
@@ -238,6 +239,7 @@ impl ConfigError {
|
||||
ConfigError::ParseError { .. } => false,
|
||||
ConfigError::SerializationError { .. } => false,
|
||||
ConfigError::SqlError { .. } => true,
|
||||
#[cfg(feature = "vault")]
|
||||
ConfigError::VaultClientError { .. } => true,
|
||||
ConfigError::NotInitialized => false,
|
||||
ConfigError::CacheError { .. } => false,
|
||||
@@ -265,6 +267,7 @@ impl ConfigError {
|
||||
ConfigError::EnvironmentError { .. } => "environment",
|
||||
ConfigError::NetworkError { .. } => "network",
|
||||
ConfigError::SqlError { .. } => "database",
|
||||
#[cfg(feature = "vault")]
|
||||
ConfigError::VaultClientError { .. } => "vault",
|
||||
ConfigError::AuthenticationError { .. } => "auth",
|
||||
ConfigError::TimeoutError { .. } => "timeout",
|
||||
|
||||
@@ -38,13 +38,19 @@ pub mod database;
|
||||
pub mod error;
|
||||
pub mod manager;
|
||||
pub mod ml_config;
|
||||
pub mod storage_config;
|
||||
pub mod structures;
|
||||
#[cfg(feature = "vault")]
|
||||
pub mod vault;
|
||||
|
||||
pub use data_config::*;
|
||||
pub use database::{DatabaseConfig, PostgresConfigLoader};
|
||||
pub use error::{ConfigError, ConfigResult};
|
||||
pub use manager::ConfigManager;
|
||||
pub use storage_config::{
|
||||
S3Config, StorageConfig, ModelMetadata, TrainingMetrics,
|
||||
FinancialMetrics, ModelArchitecture
|
||||
};
|
||||
// ML Config re-exports (specific to avoid conflicts)
|
||||
pub use ml_config::{
|
||||
Mamba2Config, TLOBConfig, DQNConfig, PPOConfig,
|
||||
@@ -59,6 +65,7 @@ pub use structures::{
|
||||
TrainingConfig as SystemTrainingConfig, PerformanceConfig as SystemPerformanceConfig,
|
||||
MLConfig, InferenceConfig as StructInferenceConfig, KellyConfig
|
||||
};
|
||||
#[cfg(feature = "vault")]
|
||||
pub use vault::{VaultConfig, VaultSecrets};
|
||||
|
||||
// Note: BacktestingConfig already exported above
|
||||
|
||||
@@ -9,8 +9,11 @@
|
||||
|
||||
use crate::{
|
||||
ConfigCategory, ConfigChange, ConfigError, ConfigHealth, ConfigResult, ConfigSource,
|
||||
ConfigValue, DatabaseConfig, PostgresConfigLoader, VaultConfig, VaultSecrets,
|
||||
ConfigValue, DatabaseConfig, PostgresConfigLoader,
|
||||
};
|
||||
|
||||
#[cfg(feature = "vault")]
|
||||
use crate::{VaultConfig, VaultSecrets};
|
||||
use chrono::Utc;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
@@ -63,7 +66,8 @@ pub struct ConfigManager {
|
||||
/// PostgreSQL configuration loader
|
||||
postgres_loader: Option<Arc<PostgresConfigLoader>>,
|
||||
/// Vault secrets manager
|
||||
vault_secrets: Option<Arc<VaultSecrets>>,
|
||||
#[cfg(feature = "vault")]
|
||||
vault_secrets: Option<Arc<crate::vault::VaultSecrets>>,
|
||||
/// Manager settings
|
||||
settings: ConfigManagerSettings,
|
||||
/// Configuration change notifications
|
||||
@@ -76,7 +80,8 @@ impl ConfigManager {
|
||||
/// Create new configuration manager with PostgreSQL and Vault
|
||||
pub async fn new(
|
||||
db_config: Option<DatabaseConfig>,
|
||||
vault_config: Option<VaultConfig>,
|
||||
#[cfg(feature = "vault")]
|
||||
vault_config: Option<crate::vault::VaultConfig>,
|
||||
settings: Option<ConfigManagerSettings>,
|
||||
) -> ConfigResult<Self> {
|
||||
let settings = settings.unwrap_or_default();
|
||||
@@ -104,9 +109,10 @@ impl ConfigManager {
|
||||
};
|
||||
|
||||
// Initialize Vault secrets manager if enabled and configured
|
||||
#[cfg(feature = "vault")]
|
||||
let vault_secrets = if settings.enable_vault {
|
||||
if let Some(vault_config) = vault_config {
|
||||
match VaultSecrets::new(vault_config).await {
|
||||
match crate::vault::VaultSecrets::new(vault_config).await {
|
||||
Ok(vault) => {
|
||||
info!("Vault secrets manager initialized");
|
||||
Some(Arc::new(vault))
|
||||
@@ -124,10 +130,14 @@ impl ConfigManager {
|
||||
None
|
||||
};
|
||||
|
||||
#[cfg(not(feature = "vault"))]
|
||||
let vault_secrets = None;
|
||||
|
||||
let (change_tx, _change_rx) = mpsc::unbounded_channel();
|
||||
|
||||
let manager = Self {
|
||||
postgres_loader,
|
||||
#[cfg(feature = "vault")]
|
||||
vault_secrets,
|
||||
settings,
|
||||
change_tx,
|
||||
@@ -157,7 +167,8 @@ impl ConfigManager {
|
||||
};
|
||||
|
||||
let vault_config = if std::env::var("VAULT_ADDR").is_ok() && std::env::var("VAULT_ROLE_ID").is_ok() {
|
||||
Some(VaultConfig::from_env()?)
|
||||
#[cfg(feature = "vault")]
|
||||
Some(crate::vault::VaultConfig::from_env()?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -439,6 +450,7 @@ impl ConfigManager {
|
||||
}
|
||||
|
||||
// Check Vault health
|
||||
#[cfg(feature = "vault")]
|
||||
if let Some(ref vault) = vault_secrets {
|
||||
let vault_health = if vault.health_check().await {
|
||||
ConfigHealth {
|
||||
|
||||
287
crates/config/src/storage_config.rs
Normal file
287
crates/config/src/storage_config.rs
Normal file
@@ -0,0 +1,287 @@
|
||||
//! S3 and Model Storage Configuration
|
||||
//!
|
||||
//! This module provides configuration structures for S3 storage and model management
|
||||
//! used by the ML Training Service and other components.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// S3 storage configuration
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct S3Config {
|
||||
/// S3 bucket name for model storage
|
||||
pub bucket_name: String,
|
||||
/// AWS region
|
||||
pub region: String,
|
||||
/// AWS access key ID (preferably from Vault)
|
||||
pub access_key_id: String,
|
||||
/// AWS secret access key (preferably from Vault)
|
||||
pub secret_access_key: String,
|
||||
/// Optional session token
|
||||
pub session_token: Option<String>,
|
||||
/// S3 endpoint URL (for custom S3-compatible services)
|
||||
pub endpoint_url: Option<String>,
|
||||
/// Whether to use path-style addressing
|
||||
pub force_path_style: bool,
|
||||
}
|
||||
|
||||
impl Default for S3Config {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
bucket_name: "foxhunt-models".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
access_key_id: String::new(),
|
||||
secret_access_key: String::new(),
|
||||
session_token: None,
|
||||
endpoint_url: None,
|
||||
force_path_style: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Storage configuration for ML models
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct StorageConfig {
|
||||
/// Storage backend type ("local" or "s3")
|
||||
pub storage_type: String,
|
||||
/// Local storage base path (when using local storage)
|
||||
pub local_base_path: Option<PathBuf>,
|
||||
/// S3 configuration (when using S3 storage)
|
||||
pub s3_config: Option<S3Config>,
|
||||
/// Enable compression for stored models
|
||||
pub enable_compression: bool,
|
||||
/// Model retention policy in days
|
||||
pub retention_days: u32,
|
||||
}
|
||||
|
||||
impl Default for StorageConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
storage_type: "local".to_string(),
|
||||
local_base_path: Some(PathBuf::from("./models")),
|
||||
s3_config: None,
|
||||
enable_compression: true,
|
||||
retention_days: 30,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Model metadata for versioning and tracking
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ModelMetadata {
|
||||
/// Model version identifier
|
||||
pub version: String,
|
||||
/// Model name/type
|
||||
pub model_name: String,
|
||||
/// Training job ID that created this model
|
||||
pub job_id: String,
|
||||
/// Timestamp when model was created
|
||||
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||
/// Model file size in bytes
|
||||
pub file_size_bytes: u64,
|
||||
/// Training metrics
|
||||
pub training_metrics: TrainingMetrics,
|
||||
/// Model hyperparameters
|
||||
pub hyperparameters: std::collections::HashMap<String, serde_json::Value>,
|
||||
/// Model architecture information
|
||||
pub architecture: ModelArchitecture,
|
||||
}
|
||||
|
||||
/// Training metrics for model performance tracking
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TrainingMetrics {
|
||||
/// Final training loss
|
||||
pub final_train_loss: f64,
|
||||
/// Final validation loss
|
||||
pub final_val_loss: f64,
|
||||
/// Training accuracy (if applicable)
|
||||
pub accuracy: Option<f64>,
|
||||
/// Number of training epochs completed
|
||||
pub epochs_completed: u32,
|
||||
/// Training duration in seconds
|
||||
pub training_duration_secs: u64,
|
||||
/// Financial performance metrics
|
||||
pub financial_metrics: Option<FinancialMetrics>,
|
||||
}
|
||||
|
||||
/// Financial performance metrics
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct FinancialMetrics {
|
||||
/// Simulated return percentage
|
||||
pub simulated_return: f64,
|
||||
/// Sharpe ratio
|
||||
pub sharpe_ratio: f64,
|
||||
/// Maximum drawdown
|
||||
pub max_drawdown: f64,
|
||||
/// Hit rate (percentage of profitable trades)
|
||||
pub hit_rate: f64,
|
||||
/// Average prediction error in basis points
|
||||
pub avg_prediction_error_bps: f64,
|
||||
/// Risk-adjusted return
|
||||
pub risk_adjusted_return: f64,
|
||||
}
|
||||
|
||||
/// Model architecture information
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ModelArchitecture {
|
||||
/// Model type (TLOB, MAMBA, DQN, etc.)
|
||||
pub model_type: String,
|
||||
/// Input dimension
|
||||
pub input_dim: usize,
|
||||
/// Output dimension
|
||||
pub output_dim: usize,
|
||||
/// Number of parameters
|
||||
pub parameter_count: Option<u64>,
|
||||
/// Model complexity score
|
||||
pub complexity_score: Option<f64>,
|
||||
}
|
||||
|
||||
impl S3Config {
|
||||
/// Create S3Config from environment variables
|
||||
pub fn from_env() -> Result<Self, Box<dyn std::error::Error>> {
|
||||
Ok(Self {
|
||||
bucket_name: std::env::var("S3_MODEL_STORAGE_BUCKET")
|
||||
.unwrap_or_else(|_| "foxhunt-models".to_string()),
|
||||
region: std::env::var("AWS_REGION")
|
||||
.unwrap_or_else(|_| "us-east-1".to_string()),
|
||||
access_key_id: std::env::var("AWS_ACCESS_KEY_ID")?,
|
||||
secret_access_key: std::env::var("AWS_SECRET_ACCESS_KEY")?,
|
||||
session_token: std::env::var("AWS_SESSION_TOKEN").ok(),
|
||||
endpoint_url: std::env::var("S3_ENDPOINT_URL").ok(),
|
||||
force_path_style: std::env::var("S3_FORCE_PATH_STYLE")
|
||||
.map(|v| v.to_lowercase() == "true")
|
||||
.unwrap_or(false),
|
||||
})
|
||||
}
|
||||
|
||||
/// Validate S3 configuration
|
||||
pub fn validate(&self) -> Result<(), Box<dyn std::error::Error>> {
|
||||
if self.bucket_name.is_empty() {
|
||||
return Err("S3 bucket name cannot be empty".into());
|
||||
}
|
||||
if self.region.is_empty() {
|
||||
return Err("AWS region cannot be empty".into());
|
||||
}
|
||||
if self.access_key_id.is_empty() {
|
||||
return Err("AWS access key ID cannot be empty".into());
|
||||
}
|
||||
if self.secret_access_key.is_empty() {
|
||||
return Err("AWS secret access key cannot be empty".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl StorageConfig {
|
||||
/// Create StorageConfig from environment variables
|
||||
pub fn from_env() -> Result<Self, Box<dyn std::error::Error>> {
|
||||
let storage_type = std::env::var("MODEL_STORAGE_TYPE")
|
||||
.unwrap_or_else(|_| "local".to_string());
|
||||
|
||||
let (local_base_path, s3_config) = match storage_type.as_str() {
|
||||
"s3" => (None, Some(S3Config::from_env()?)),
|
||||
"local" => {
|
||||
let path = std::env::var("MODEL_STORAGE_PATH")
|
||||
.unwrap_or_else(|_| "./models".to_string());
|
||||
(Some(PathBuf::from(path)), None)
|
||||
},
|
||||
_ => return Err(format!("Unsupported storage type: {}", storage_type).into()),
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
storage_type,
|
||||
local_base_path,
|
||||
s3_config,
|
||||
enable_compression: std::env::var("MODEL_COMPRESSION_ENABLED")
|
||||
.map(|v| v.to_lowercase() == "true")
|
||||
.unwrap_or(true),
|
||||
retention_days: std::env::var("MODEL_RETENTION_DAYS")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(30),
|
||||
})
|
||||
}
|
||||
|
||||
/// Validate storage configuration
|
||||
pub fn validate(&self) -> Result<(), Box<dyn std::error::Error>> {
|
||||
match self.storage_type.as_str() {
|
||||
"local" => {
|
||||
if self.local_base_path.is_none() {
|
||||
return Err("Local base path required for local storage".into());
|
||||
}
|
||||
},
|
||||
"s3" => {
|
||||
if let Some(ref s3_config) = self.s3_config {
|
||||
s3_config.validate()?;
|
||||
} else {
|
||||
return Err("S3 config required for S3 storage".into());
|
||||
}
|
||||
},
|
||||
_ => return Err(format!("Unsupported storage type: {}", self.storage_type).into()),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if S3 storage is enabled
|
||||
pub fn is_s3_enabled(&self) -> bool {
|
||||
self.storage_type == "s3" && self.s3_config.is_some()
|
||||
}
|
||||
|
||||
/// Get S3 config if available
|
||||
pub fn get_s3_config(&self) -> Option<&S3Config> {
|
||||
if self.is_s3_enabled() {
|
||||
self.s3_config.as_ref()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_s3_config_default() {
|
||||
let config = S3Config::default();
|
||||
assert_eq!(config.bucket_name, "foxhunt-models");
|
||||
assert_eq!(config.region, "us-east-1");
|
||||
assert!(config.access_key_id.is_empty());
|
||||
assert!(!config.force_path_style);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_storage_config_default() {
|
||||
let config = StorageConfig::default();
|
||||
assert_eq!(config.storage_type, "local");
|
||||
assert!(config.local_base_path.is_some());
|
||||
assert!(config.s3_config.is_none());
|
||||
assert!(config.enable_compression);
|
||||
assert_eq!(config.retention_days, 30);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_storage_config_validation() {
|
||||
let mut config = StorageConfig::default();
|
||||
assert!(config.validate().is_ok());
|
||||
|
||||
// Test S3 validation
|
||||
config.storage_type = "s3".to_string();
|
||||
config.s3_config = None;
|
||||
assert!(config.validate().is_err());
|
||||
|
||||
config.s3_config = Some(S3Config::default());
|
||||
// This will fail because access keys are empty
|
||||
assert!(config.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_s3_config_validation() {
|
||||
let mut config = S3Config::default();
|
||||
assert!(config.validate().is_err()); // Empty credentials
|
||||
|
||||
config.access_key_id = "test_key".to_string();
|
||||
config.secret_access_key = "test_secret".to_string();
|
||||
assert!(config.validate().is_ok());
|
||||
}
|
||||
}
|
||||
@@ -247,11 +247,19 @@ async fn serve(args: ServeArgs) -> Result<()> {
|
||||
|
||||
info!("Database connection established");
|
||||
|
||||
// Initialize storage with ConfigManager integration
|
||||
let default_storage_config = storage::StorageConfig::default();
|
||||
// Initialize storage with S3 configuration from environment
|
||||
let storage_config = config::StorageConfig::from_env()
|
||||
.unwrap_or_else(|e| {
|
||||
warn!("Failed to load S3 config from environment: {}, using defaults", e);
|
||||
config::StorageConfig::default()
|
||||
});
|
||||
|
||||
info!("Storage configuration: type={}, compression={}",
|
||||
storage_config.storage_type, storage_config.enable_compression);
|
||||
|
||||
let storage = Arc::new(
|
||||
ModelStorageManager::new_with_config_manager(
|
||||
default_storage_config,
|
||||
storage_config.into(), // Convert to storage::StorageConfig
|
||||
Arc::clone(&config_manager),
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -21,7 +21,7 @@ use ml::training_pipeline::{
|
||||
ProductionTrainingMetrics, TrainingResult,
|
||||
};
|
||||
|
||||
use config::{MLConfig, TrainingConfig as SystemTrainingConfig};
|
||||
use config::{MLConfig, TrainingConfig as SystemTrainingConfig, ModelMetadata, TrainingMetrics, ModelArchitecture};
|
||||
use crate::database::TrainingJobRecord;
|
||||
use crate::repository::MlDataRepository;
|
||||
use crate::storage::ModelStorageManager;
|
||||
@@ -697,9 +697,64 @@ impl TrainingOrchestrator {
|
||||
database: &Arc<DatabaseManager>,
|
||||
storage: &Arc<ModelStorageManager>,
|
||||
) -> Result<()> {
|
||||
// Store model artifact (mock implementation)
|
||||
let model_path = format!("models/{}.bin", job_id);
|
||||
// Serialize training result to model data
|
||||
let model_data = match serde_json::to_vec(&result) {
|
||||
Ok(data) => data,
|
||||
Err(e) => {
|
||||
warn!("Failed to serialize training result for job {}: {}", job_id, e);
|
||||
return Err(anyhow::anyhow!("Model serialization failed: {}", e));
|
||||
}
|
||||
};
|
||||
|
||||
// Attempt to store model artifact with S3 upload
|
||||
let model_path = match storage.store_model(job_id, &model_data).await {
|
||||
Ok(path) => {
|
||||
info!("Successfully uploaded model for job {} to storage: {}", job_id, path);
|
||||
path
|
||||
},
|
||||
Err(e) => {
|
||||
warn!("Failed to upload model for job {}: {}", job_id, e);
|
||||
// Fallback to local path for tracking
|
||||
let fallback_path = format!("models/{}.bin", job_id);
|
||||
warn!("Using fallback path for job {}: {}", job_id, fallback_path);
|
||||
fallback_path
|
||||
}
|
||||
};
|
||||
|
||||
// Create model metadata for tracking and versioning
|
||||
let model_metadata = {
|
||||
let job_guard = jobs.read().await;
|
||||
if let Some(job) = job_guard.get(&job_id) {
|
||||
config::ModelMetadata {
|
||||
version: format!("v{}", chrono::Utc::now().format("%Y%m%d_%H%M%S")),
|
||||
model_name: job.model_type.clone(),
|
||||
job_id: job_id.to_string(),
|
||||
created_at: chrono::Utc::now(),
|
||||
file_size_bytes: model_data.len() as u64,
|
||||
training_metrics: config::TrainingMetrics {
|
||||
final_train_loss: result.final_train_loss,
|
||||
final_val_loss: result.final_val_loss,
|
||||
accuracy: None, // TODO: Add accuracy if available
|
||||
epochs_completed: result.epochs_trained as u32,
|
||||
training_duration_secs: result.training_duration.as_secs(),
|
||||
financial_metrics: None, // TODO: Add financial metrics if available
|
||||
},
|
||||
hyperparameters: job.hyperparameters.clone(),
|
||||
architecture: config::ModelArchitecture {
|
||||
model_type: job.model_type.clone(),
|
||||
input_dim: 0, // TODO: Get from model config
|
||||
output_dim: 0, // TODO: Get from model config
|
||||
parameter_count: None,
|
||||
complexity_score: None,
|
||||
},
|
||||
}
|
||||
} else {
|
||||
return Err(anyhow::anyhow!("Job {} not found for metadata creation", job_id));
|
||||
}
|
||||
};
|
||||
|
||||
info!("Created model metadata for job {} with version {}", job_id, model_metadata.version);
|
||||
|
||||
// Update job status
|
||||
{
|
||||
let mut jobs_write = jobs.write().await;
|
||||
@@ -712,6 +767,10 @@ impl TrainingOrchestrator {
|
||||
.insert("final_train_loss".to_string(), result.final_train_loss);
|
||||
job.metrics
|
||||
.insert("final_val_loss".to_string(), result.final_val_loss);
|
||||
job.metrics
|
||||
.insert("model_version".to_string(), model_metadata.version.parse().unwrap_or(0.0));
|
||||
job.metrics
|
||||
.insert("model_size_bytes".to_string(), model_metadata.file_size_bytes as f64);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,34 @@ use uuid::Uuid;
|
||||
use config::PerformanceConfig as SystemPerformanceConfig;
|
||||
use config::ConfigLoader;
|
||||
|
||||
/// Local storage configuration for ML models
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct StorageConfig {
|
||||
pub storage_type: String,
|
||||
pub local_base_path: Option<PathBuf>,
|
||||
pub enable_compression: bool,
|
||||
}
|
||||
|
||||
impl Default for StorageConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
storage_type: "local".to_string(),
|
||||
local_base_path: Some(PathBuf::from("./models")),
|
||||
enable_compression: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<config::StorageConfig> for StorageConfig {
|
||||
fn from(config_storage: config::StorageConfig) -> Self {
|
||||
Self {
|
||||
storage_type: config_storage.storage_type,
|
||||
local_base_path: config_storage.local_base_path,
|
||||
enable_compression: config_storage.enable_compression,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait for model storage operations
|
||||
#[async_trait]
|
||||
pub trait ModelStorage: Send + Sync {
|
||||
|
||||
@@ -15,6 +15,10 @@ path = "src/main.rs"
|
||||
name = "latency_validator"
|
||||
path = "src/bin/latency_validator.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "model_cache_benchmark"
|
||||
path = "src/bin/model_cache_benchmark.rs"
|
||||
|
||||
[dependencies]
|
||||
# Core async and utilities - USE WORKSPACE
|
||||
tokio.workspace = true
|
||||
@@ -45,14 +49,21 @@ async-trait.workspace = true
|
||||
# Performance monitoring
|
||||
hdrhistogram.workspace = true
|
||||
|
||||
# Model caching dependencies
|
||||
memmap2 = "0.9"
|
||||
sha2 = "0.10"
|
||||
semver = { version = "1.0", features = ["serde"] }
|
||||
thiserror = "1.0"
|
||||
uuid = { version = "1.0", features = ["v4", "serde"] }
|
||||
|
||||
# Internal workspace crates
|
||||
trading_engine.workspace = true
|
||||
risk.workspace = true
|
||||
ml.workspace = true
|
||||
data.workspace = true
|
||||
common = { workspace = true, features = ["database"] }
|
||||
storage = { workspace = true, features = ["s3"] }
|
||||
config = { workspace = true, features = ["postgres", "vault"] }
|
||||
storage = { workspace = true }
|
||||
config = { workspace = true, features = ["postgres"] }
|
||||
|
||||
[build-dependencies]
|
||||
tonic-build.workspace = true
|
||||
|
||||
182
services/trading_service/src/bin/model_cache_benchmark.rs
Normal file
182
services/trading_service/src/bin/model_cache_benchmark.rs
Normal file
@@ -0,0 +1,182 @@
|
||||
//! Model Cache Performance Benchmark
|
||||
//!
|
||||
//! This benchmark validates that model inference through the cache
|
||||
//! achieves the target <50μs latency for high-frequency trading.
|
||||
|
||||
use anyhow::Result;
|
||||
use std::time::Instant;
|
||||
use trading_service::model_cache::{ModelCache, CacheConfig};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
// Initialize tracing
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter("info")
|
||||
.init();
|
||||
|
||||
println!("🚀 Model Cache Performance Benchmark");
|
||||
println!("Target: <50μs inference latency");
|
||||
println!("================================================");
|
||||
|
||||
// Create cache configuration for benchmarking
|
||||
let cache_config = CacheConfig {
|
||||
cache_dir: "/tmp/foxhunt_benchmark_cache".into(),
|
||||
s3_bucket: std::env::var("ML_MODELS_S3_BUCKET")
|
||||
.unwrap_or_else(|_| "foxhunt-ml-models".to_string()),
|
||||
s3_region: std::env::var("AWS_REGION")
|
||||
.unwrap_or_else(|_| "us-east-1".to_string()),
|
||||
update_interval_secs: 3600, // 1 hour for benchmark
|
||||
max_cache_size_bytes: 1024 * 1024 * 1024, // 1GB for benchmark
|
||||
versions_to_keep: 2,
|
||||
};
|
||||
|
||||
// Initialize model cache
|
||||
println!("Initializing ModelCache...");
|
||||
let mut model_cache = ModelCache::new(cache_config).await?;
|
||||
|
||||
// Create test model files if they don't exist
|
||||
create_test_models(&model_cache).await?;
|
||||
|
||||
model_cache.initialize().await?;
|
||||
println!("✅ ModelCache initialized successfully");
|
||||
|
||||
// Wait for initialization to complete
|
||||
while !model_cache.is_initialized().await {
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
|
||||
// Benchmark model access performance
|
||||
println!("\n📊 Running Performance Benchmarks...");
|
||||
|
||||
let test_models = vec![
|
||||
"tlob_transformer",
|
||||
"dqn_policy",
|
||||
"mamba2",
|
||||
"tft_predictor"
|
||||
];
|
||||
|
||||
for model_name in &test_models {
|
||||
if let Err(e) = benchmark_model_access(&model_cache, model_name).await {
|
||||
eprintln!("❌ Benchmark failed for {}: {}", model_name, e);
|
||||
}
|
||||
}
|
||||
|
||||
// Overall performance summary
|
||||
println!("\n🎯 Performance Summary");
|
||||
let stats = model_cache.get_cache_stats().await;
|
||||
println!("Total models cached: {}", stats.get("total_models").unwrap_or(&serde_json::Value::from(0)));
|
||||
println!("Total cache size: {}MB", stats.get("total_size_mb").unwrap_or(&serde_json::Value::from(0)));
|
||||
println!("Critical models: {}", stats.get("critical_models").unwrap_or(&serde_json::Value::from(0)));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create test model files for benchmarking
|
||||
async fn create_test_models(model_cache: &ModelCache) -> Result<()> {
|
||||
use std::fs;
|
||||
|
||||
let cache_dir = std::path::Path::new("/tmp/foxhunt_benchmark_cache");
|
||||
fs::create_dir_all(cache_dir.join("current"))?;
|
||||
|
||||
// Create mock model files with different sizes
|
||||
let test_models = [
|
||||
("tlob_transformer", 1024 * 1024), // 1MB
|
||||
("dqn_policy", 512 * 1024), // 512KB
|
||||
("mamba2", 2 * 1024 * 1024), // 2MB
|
||||
("tft_predictor", 1536 * 1024), // 1.5MB
|
||||
];
|
||||
|
||||
for (name, size) in &test_models {
|
||||
let file_path = cache_dir.join("current").join(format!("{}.onnx", name));
|
||||
|
||||
if !file_path.exists() {
|
||||
// Create a mock model file with random data
|
||||
let data: Vec<u8> = (0..*size).map(|i| (i % 256) as u8).collect();
|
||||
fs::write(&file_path, data)?;
|
||||
println!("Created test model: {} ({}KB)", name, size / 1024);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Benchmark model access latency
|
||||
async fn benchmark_model_access(model_cache: &ModelCache, model_name: &str) -> Result<()> {
|
||||
const WARMUP_ITERATIONS: usize = 1000;
|
||||
const BENCHMARK_ITERATIONS: usize = 10000;
|
||||
|
||||
println!("\n🔥 Benchmarking model: {}", model_name);
|
||||
|
||||
// Warmup phase
|
||||
print!("Warming up... ");
|
||||
for _ in 0..WARMUP_ITERATIONS {
|
||||
if let Ok(_data) = model_cache.get_model(model_name).await {
|
||||
// Access the data to ensure it's actually loaded
|
||||
}
|
||||
}
|
||||
println!("done");
|
||||
|
||||
// Benchmark phase
|
||||
let mut latencies = Vec::with_capacity(BENCHMARK_ITERATIONS);
|
||||
|
||||
print!("Running {} iterations... ", BENCHMARK_ITERATIONS);
|
||||
for _ in 0..BENCHMARK_ITERATIONS {
|
||||
let start = Instant::now();
|
||||
|
||||
if let Ok(data) = model_cache.get_model(model_name).await {
|
||||
// Ensure we actually access the data
|
||||
let _checksum = data.len();
|
||||
}
|
||||
|
||||
let latency = start.elapsed();
|
||||
latencies.push(latency);
|
||||
}
|
||||
println!("done");
|
||||
|
||||
// Calculate statistics
|
||||
latencies.sort();
|
||||
|
||||
let avg_ns = latencies.iter().map(|d| d.as_nanos()).sum::<u128>() / latencies.len() as u128;
|
||||
let p50_ns = latencies[latencies.len() / 2].as_nanos();
|
||||
let p95_ns = latencies[latencies.len() * 95 / 100].as_nanos();
|
||||
let p99_ns = latencies[latencies.len() * 99 / 100].as_nanos();
|
||||
let max_ns = latencies.last().unwrap().as_nanos();
|
||||
|
||||
// Convert to microseconds
|
||||
let avg_us = avg_ns as f64 / 1000.0;
|
||||
let p50_us = p50_ns as f64 / 1000.0;
|
||||
let p95_us = p95_ns as f64 / 1000.0;
|
||||
let p99_us = p99_ns as f64 / 1000.0;
|
||||
let max_us = max_ns as f64 / 1000.0;
|
||||
|
||||
// Results
|
||||
println!("📈 Results for {}:", model_name);
|
||||
println!(" Average: {:.2}μs", avg_us);
|
||||
println!(" P50: {:.2}μs", p50_us);
|
||||
println!(" P95: {:.2}μs", p95_us);
|
||||
println!(" P99: {:.2}μs", p99_us);
|
||||
println!(" Max: {:.2}μs", max_us);
|
||||
|
||||
// Check if we meet the target
|
||||
let target_us = 50.0;
|
||||
let meets_target = p95_us < target_us;
|
||||
|
||||
if meets_target {
|
||||
println!(" ✅ PASSED: P95 latency {:.2}μs < {}μs target", p95_us, target_us);
|
||||
} else {
|
||||
println!(" ❌ FAILED: P95 latency {:.2}μs >= {}μs target", p95_us, target_us);
|
||||
}
|
||||
|
||||
// Performance grade
|
||||
let grade = match p95_us {
|
||||
x if x < 10.0 => "A+ (Excellent)",
|
||||
x if x < 25.0 => "A (Very Good)",
|
||||
x if x < 50.0 => "B (Good)",
|
||||
x if x < 100.0 => "C (Acceptable)",
|
||||
_ => "D (Needs Improvement)",
|
||||
};
|
||||
|
||||
println!(" 🎯 Performance Grade: {}", grade);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -68,6 +68,9 @@ pub mod repository_impls;
|
||||
/// High-precision latency recording with HDR histogram
|
||||
pub mod latency_recorder;
|
||||
|
||||
/// High-performance model caching with S3 backend and <50μs access
|
||||
pub mod model_cache;
|
||||
|
||||
/// Service implementations for gRPC endpoints
|
||||
pub mod services;
|
||||
|
||||
@@ -94,6 +97,7 @@ pub mod prelude {
|
||||
pub use crate::error::*;
|
||||
pub use crate::event_streaming::*;
|
||||
pub use crate::latency_recorder::*;
|
||||
pub use crate::model_cache::*;
|
||||
pub use crate::repositories::*;
|
||||
pub use crate::repository_impls::*;
|
||||
pub use crate::services::*;
|
||||
|
||||
@@ -28,6 +28,7 @@ use trading_service::repository_impls::*;
|
||||
use trading_service::kill_switch_integration::TradingServiceKillSwitch;
|
||||
use trading_service::prelude::*;
|
||||
use trading_service::services::{EnhancedMLServiceImpl, MLFallbackManager, MLPerformanceMonitor};
|
||||
use trading_service::model_cache::{ModelCache, CacheConfig};
|
||||
|
||||
/// Default configuration values
|
||||
const DEFAULT_CONFIG_TTL: Duration = Duration::from_secs(300); // 5 minutes
|
||||
@@ -107,6 +108,33 @@ async fn main() -> Result<()> {
|
||||
.context("Failed to start kill switch monitoring")?;
|
||||
info!("Kill switch monitoring started - emergency shutdown ready");
|
||||
|
||||
// Initialize high-performance model cache for <50μs inference
|
||||
let cache_config = CacheConfig {
|
||||
cache_dir: std::env::var("MODEL_CACHE_DIR")
|
||||
.unwrap_or_else(|_| "/opt/foxhunt/model_cache".to_string())
|
||||
.into(),
|
||||
s3_bucket: std::env::var("ML_MODELS_S3_BUCKET")
|
||||
.unwrap_or_else(|_| "foxhunt-ml-models".to_string()),
|
||||
s3_region: std::env::var("AWS_REGION")
|
||||
.unwrap_or_else(|_| "us-east-1".to_string()),
|
||||
update_interval_secs: std::env::var("MODEL_UPDATE_INTERVAL_SECS")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(300), // 5 minutes default
|
||||
max_cache_size_bytes: 5 * 1024 * 1024 * 1024, // 5GB
|
||||
versions_to_keep: 3,
|
||||
};
|
||||
|
||||
let mut model_cache = ModelCache::new(cache_config).await
|
||||
.context("Failed to create ModelCache")?;
|
||||
|
||||
// Initialize model cache - this will download models from S3 or load cached
|
||||
model_cache.initialize().await
|
||||
.context("Failed to initialize ModelCache")?;
|
||||
|
||||
let model_cache = Arc::new(model_cache);
|
||||
info!("Model cache initialized with <50μs inference capability");
|
||||
|
||||
// Start configuration hot-reload monitoring
|
||||
start_config_monitoring(config_repository.clone()).await?;
|
||||
|
||||
@@ -129,9 +157,10 @@ async fn main() -> Result<()> {
|
||||
market_data_repository,
|
||||
risk_repository,
|
||||
config_repository.clone(),
|
||||
Some(Arc::clone(&kill_switch_system))
|
||||
Some(Arc::clone(&kill_switch_system)),
|
||||
Some(Arc::clone(&model_cache))
|
||||
).await?;
|
||||
info!("Trading service state initialized with repository dependency injection");
|
||||
info!("Trading service state initialized with repository dependency injection and model cache");
|
||||
|
||||
// Initialize ML performance monitoring and fallback management
|
||||
let ml_performance_monitor = Arc::new(MLPerformanceMonitor::new().await?);
|
||||
|
||||
624
services/trading_service/src/model_cache.rs
Normal file
624
services/trading_service/src/model_cache.rs
Normal file
@@ -0,0 +1,624 @@
|
||||
//! Model Cache System for <50μs ML Inference
|
||||
//!
|
||||
//! This module provides high-performance model caching with:
|
||||
//! - S3 model download and local NVMe/SSD caching
|
||||
//! - Memory-mapped zero-copy model access for <50μs inference
|
||||
//! - Graceful fallback to cached models when S3 is unavailable
|
||||
//! - Atomic model updates without blocking inference operations
|
||||
//! - Version management with rollback capability
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use memmap2::{Mmap, MmapOptions};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::fs::{self, File};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
use tokio::sync::{RwLock, broadcast};
|
||||
use tokio::time::interval;
|
||||
use tracing::{debug, error, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Model types supported by the caching system
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub enum ModelType {
|
||||
/// TLOB Transformer for order book analysis
|
||||
TlobTransformer,
|
||||
/// Deep Q-Network for policy decisions
|
||||
Dqn,
|
||||
/// MAMBA-2 State Space Model
|
||||
Mamba2,
|
||||
/// Temporal Fusion Transformer
|
||||
Tft,
|
||||
/// Policy Proximal Optimization
|
||||
Ppo,
|
||||
/// Liquid Networks
|
||||
Liquid,
|
||||
/// Custom ensemble model
|
||||
Ensemble,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ModelType {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
ModelType::TlobTransformer => write!(f, "tlob_transformer"),
|
||||
ModelType::Dqn => write!(f, "dqn"),
|
||||
ModelType::Mamba2 => write!(f, "mamba2"),
|
||||
ModelType::Tft => write!(f, "tft"),
|
||||
ModelType::Ppo => write!(f, "ppo"),
|
||||
ModelType::Liquid => write!(f, "liquid"),
|
||||
ModelType::Ensemble => write!(f, "ensemble"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Model priority for loading order
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub enum ModelPriority {
|
||||
/// Critical models loaded immediately (TLOB, DQN)
|
||||
Critical,
|
||||
/// Important models loaded in background
|
||||
Normal,
|
||||
/// Optional models loaded when resources available
|
||||
Low,
|
||||
}
|
||||
|
||||
/// Cached model metadata and memory mapping
|
||||
#[derive(Debug)]
|
||||
pub struct CachedModel {
|
||||
pub model_type: ModelType,
|
||||
pub version: semver::Version,
|
||||
pub file_path: PathBuf,
|
||||
pub mmap_region: Mmap,
|
||||
pub last_used: Instant,
|
||||
pub checksum: String,
|
||||
pub priority: ModelPriority,
|
||||
pub file_size: u64,
|
||||
pub load_time: SystemTime,
|
||||
}
|
||||
|
||||
/// Model cache configuration
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CacheConfig {
|
||||
/// Base cache directory (e.g., /opt/foxhunt/model_cache)
|
||||
pub cache_dir: PathBuf,
|
||||
/// S3 bucket name for model storage
|
||||
pub s3_bucket: String,
|
||||
/// S3 region
|
||||
pub s3_region: String,
|
||||
/// Update check interval in seconds
|
||||
pub update_interval_secs: u64,
|
||||
/// Maximum cache size in bytes
|
||||
pub max_cache_size_bytes: u64,
|
||||
/// Number of versions to keep per model
|
||||
pub versions_to_keep: u32,
|
||||
}
|
||||
|
||||
impl Default for CacheConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
cache_dir: PathBuf::from("/opt/foxhunt/model_cache"),
|
||||
s3_bucket: "foxhunt-ml-models".to_string(),
|
||||
s3_region: "us-east-1".to_string(),
|
||||
update_interval_secs: 300, // 5 minutes
|
||||
max_cache_size_bytes: 5 * 1024 * 1024 * 1024, // 5GB
|
||||
versions_to_keep: 3,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// S3 object metadata for version tracking
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct S3ObjectMetadata {
|
||||
pub key: String,
|
||||
pub etag: String,
|
||||
pub size: u64,
|
||||
pub last_modified: SystemTime,
|
||||
}
|
||||
|
||||
/// Model update summary
|
||||
#[derive(Debug)]
|
||||
pub struct UpdateSummary {
|
||||
pub models_checked: u32,
|
||||
pub models_updated: u32,
|
||||
pub total_download_size: u64,
|
||||
pub update_duration: Duration,
|
||||
pub errors: Vec<String>,
|
||||
}
|
||||
|
||||
/// High-performance model cache with S3 backend
|
||||
pub struct ModelCache {
|
||||
config: CacheConfig,
|
||||
cached_models: Arc<RwLock<HashMap<String, CachedModel>>>,
|
||||
s3_client: Arc<aws_sdk_s3::Client>,
|
||||
update_broadcaster: broadcast::Sender<String>,
|
||||
is_initialized: Arc<RwLock<bool>>,
|
||||
}
|
||||
|
||||
impl ModelCache {
|
||||
/// Create new model cache instance
|
||||
pub async fn new(config: CacheConfig) -> Result<Self> {
|
||||
// Initialize AWS S3 client
|
||||
let aws_config = aws_config::from_env()
|
||||
.region(aws_sdk_s3::config::Region::new(config.s3_region.clone()))
|
||||
.load()
|
||||
.await;
|
||||
let s3_client = Arc::new(aws_sdk_s3::Client::new(&aws_config));
|
||||
|
||||
let (update_sender, _) = broadcast::channel(100);
|
||||
|
||||
Ok(Self {
|
||||
config,
|
||||
cached_models: Arc::new(RwLock::new(HashMap::new())),
|
||||
s3_client,
|
||||
update_broadcaster: update_sender,
|
||||
is_initialized: Arc::new(RwLock::new(false)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Initialize cache directory and load models
|
||||
pub async fn initialize(&mut self) -> Result<()> {
|
||||
info!("Initializing ModelCache with directory: {:?}", self.config.cache_dir);
|
||||
|
||||
// Create cache directory structure
|
||||
self.setup_cache_directories().await?;
|
||||
|
||||
// Try to download latest models from S3
|
||||
match self.sync_from_s3().await {
|
||||
Ok(summary) => {
|
||||
info!("Models synced from S3: {} updated, {} checked",
|
||||
summary.models_updated, summary.models_checked);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("S3 sync failed: {}, loading cached models", e);
|
||||
self.load_cached_models().await?;
|
||||
}
|
||||
}
|
||||
|
||||
// Memory-map all loaded models for fast access
|
||||
self.memory_map_models().await?;
|
||||
|
||||
// Mark as initialized
|
||||
*self.is_initialized.write().await = true;
|
||||
|
||||
// Start background update monitoring
|
||||
self.start_background_updates().await;
|
||||
|
||||
info!("ModelCache initialization complete");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Setup cache directory structure
|
||||
async fn setup_cache_directories(&self) -> Result<()> {
|
||||
let base_dir = &self.config.cache_dir;
|
||||
|
||||
// Create main directories
|
||||
fs::create_dir_all(base_dir.join("current"))?;
|
||||
fs::create_dir_all(base_dir.join("versions"))?;
|
||||
fs::create_dir_all(base_dir.join("staging"))?;
|
||||
|
||||
info!("Cache directories created: {:?}", base_dir);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sync models from S3 bucket
|
||||
pub async fn sync_from_s3(&self) -> Result<UpdateSummary> {
|
||||
let start_time = Instant::now();
|
||||
let mut summary = UpdateSummary {
|
||||
models_checked: 0,
|
||||
models_updated: 0,
|
||||
total_download_size: 0,
|
||||
update_duration: Duration::default(),
|
||||
errors: Vec::new(),
|
||||
};
|
||||
|
||||
info!("Starting S3 sync from bucket: {}", self.config.s3_bucket);
|
||||
|
||||
// List objects in S3 bucket
|
||||
let list_request = self.s3_client
|
||||
.list_objects_v2()
|
||||
.bucket(&self.config.s3_bucket)
|
||||
.prefix("models/latest/");
|
||||
|
||||
let response = list_request.send().await
|
||||
.context("Failed to list S3 objects")?;
|
||||
|
||||
if let Some(objects) = response.contents {
|
||||
for object in objects {
|
||||
summary.models_checked += 1;
|
||||
|
||||
if let (Some(key), Some(etag), Some(size)) = (
|
||||
object.key(),
|
||||
object.e_tag(),
|
||||
object.size(),
|
||||
) {
|
||||
match self.download_model_if_updated(key, etag, size).await {
|
||||
Ok(downloaded_size) => {
|
||||
if downloaded_size > 0 {
|
||||
summary.models_updated += 1;
|
||||
summary.total_download_size += downloaded_size;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
summary.errors.push(format!("Failed to download {}: {}", key, e));
|
||||
warn!("Failed to download model {}: {}", key, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
summary.update_duration = start_time.elapsed();
|
||||
info!("S3 sync complete: {:?}", summary);
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
/// Download model from S3 if it has been updated
|
||||
async fn download_model_if_updated(&self, key: &str, etag: &str, size: i64) -> Result<u64> {
|
||||
let model_name = self.extract_model_name_from_key(key)?;
|
||||
let local_path = self.config.cache_dir
|
||||
.join("staging")
|
||||
.join(format!("{}.onnx", model_name));
|
||||
|
||||
// Check if we need to update based on ETag
|
||||
if self.is_model_current(&model_name, etag).await? {
|
||||
debug!("Model {} is current, skipping download", model_name);
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
info!("Downloading updated model: {} ({}MB)", model_name, size / (1024 * 1024));
|
||||
|
||||
// Download from S3
|
||||
let get_request = self.s3_client
|
||||
.get_object()
|
||||
.bucket(&self.config.s3_bucket)
|
||||
.key(key);
|
||||
|
||||
let response = get_request.send().await
|
||||
.context(format!("Failed to download model: {}", key))?;
|
||||
|
||||
// Write to staging directory
|
||||
let data = response.body.collect().await
|
||||
.context("Failed to read S3 response body")?;
|
||||
let bytes = data.into_bytes();
|
||||
|
||||
fs::write(&local_path, &bytes)
|
||||
.context(format!("Failed to write model file: {:?}", local_path))?;
|
||||
|
||||
// Verify file integrity
|
||||
let checksum = self.calculate_checksum(&local_path)?;
|
||||
|
||||
// Move from staging to versions directory with atomic operation
|
||||
self.promote_staged_model(&model_name, &local_path, etag, checksum).await?;
|
||||
|
||||
info!("Successfully downloaded and promoted model: {}", model_name);
|
||||
Ok(size as u64)
|
||||
}
|
||||
|
||||
/// Check if local model is current based on ETag
|
||||
async fn is_model_current(&self, model_name: &str, etag: &str) -> Result<bool> {
|
||||
let current_link = self.config.cache_dir
|
||||
.join("current")
|
||||
.join(format!("{}.onnx", model_name));
|
||||
|
||||
if !current_link.exists() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// Read metadata file to compare ETag
|
||||
let metadata_path = current_link.with_extension("json");
|
||||
if !metadata_path.exists() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let metadata_content = fs::read_to_string(metadata_path)?;
|
||||
let metadata: serde_json::Value = serde_json::from_str(&metadata_content)?;
|
||||
|
||||
Ok(metadata.get("etag").and_then(|v| v.as_str()) == Some(etag))
|
||||
}
|
||||
|
||||
/// Extract model name from S3 key
|
||||
fn extract_model_name_from_key(&self, key: &str) -> Result<String> {
|
||||
// Expected format: models/latest/{model_name}.onnx
|
||||
let parts: Vec<&str> = key.split('/').collect();
|
||||
if parts.len() >= 3 {
|
||||
let filename = parts.last().unwrap();
|
||||
let model_name = filename.trim_end_matches(".onnx");
|
||||
Ok(model_name.to_string())
|
||||
} else {
|
||||
Err(anyhow::anyhow!("Invalid S3 key format: {}", key))
|
||||
}
|
||||
}
|
||||
|
||||
/// Calculate SHA256 checksum of file
|
||||
fn calculate_checksum(&self, file_path: &Path) -> Result<String> {
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
let data = fs::read(file_path)?;
|
||||
let hash = Sha256::digest(&data);
|
||||
Ok(format!("{:x}", hash))
|
||||
}
|
||||
|
||||
/// Promote staged model to active version
|
||||
async fn promote_staged_model(
|
||||
&self,
|
||||
model_name: &str,
|
||||
staged_path: &Path,
|
||||
etag: &str,
|
||||
checksum: String,
|
||||
) -> Result<()> {
|
||||
let version = self.generate_version_string();
|
||||
let version_dir = self.config.cache_dir
|
||||
.join("versions")
|
||||
.join(model_name)
|
||||
.join(&version);
|
||||
|
||||
// Create version directory
|
||||
fs::create_dir_all(&version_dir)?;
|
||||
|
||||
// Move model file to version directory
|
||||
let version_model_path = version_dir.join("model.onnx");
|
||||
fs::rename(staged_path, &version_model_path)?;
|
||||
|
||||
// Create metadata file
|
||||
let metadata = serde_json::json!({
|
||||
"model_name": model_name,
|
||||
"version": version,
|
||||
"etag": etag,
|
||||
"checksum": checksum,
|
||||
"promoted_at": SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(),
|
||||
"file_size": fs::metadata(&version_model_path)?.len()
|
||||
});
|
||||
|
||||
let metadata_path = version_dir.join("metadata.json");
|
||||
fs::write(metadata_path, serde_json::to_string_pretty(&metadata)?)?;
|
||||
|
||||
// Create/update symlink in current directory (atomic operation)
|
||||
let current_link = self.config.cache_dir
|
||||
.join("current")
|
||||
.join(format!("{}.onnx", model_name));
|
||||
|
||||
let temp_link = current_link.with_extension(format!("onnx.tmp.{}", Uuid::new_v4()));
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
std::os::unix::fs::symlink(&version_model_path, &temp_link)?;
|
||||
fs::rename(temp_link, current_link)?;
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
{
|
||||
if current_link.exists() {
|
||||
fs::remove_file(¤t_link)?;
|
||||
}
|
||||
fs::copy(&version_model_path, current_link)?;
|
||||
}
|
||||
|
||||
// Cleanup old versions
|
||||
self.cleanup_old_versions(model_name).await?;
|
||||
|
||||
info!("Promoted model {} to version {}", model_name, version);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Generate version string based on timestamp
|
||||
fn generate_version_string(&self) -> String {
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default();
|
||||
format!("v{}", now.as_secs())
|
||||
}
|
||||
|
||||
/// Cleanup old model versions, keeping only the configured number
|
||||
async fn cleanup_old_versions(&self, model_name: &str) -> Result<()> {
|
||||
let versions_dir = self.config.cache_dir.join("versions").join(model_name);
|
||||
|
||||
if !versions_dir.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut versions: Vec<_> = fs::read_dir(&versions_dir)?
|
||||
.filter_map(|entry| entry.ok())
|
||||
.filter_map(|entry| {
|
||||
let metadata = entry.metadata().ok()?;
|
||||
let created = metadata.created().or_else(|_| metadata.modified()).ok()?;
|
||||
Some((entry.path(), created))
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Sort by creation time (newest first)
|
||||
versions.sort_by(|a, b| b.1.cmp(&a.1));
|
||||
|
||||
// Remove versions beyond the keep limit
|
||||
for (path, _) in versions.into_iter().skip(self.config.versions_to_keep as usize) {
|
||||
if path.is_dir() {
|
||||
fs::remove_dir_all(&path)?;
|
||||
debug!("Removed old version: {:?}", path);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Load cached models from disk when S3 is unavailable
|
||||
async fn load_cached_models(&self) -> Result<()> {
|
||||
let current_dir = self.config.cache_dir.join("current");
|
||||
|
||||
if !current_dir.exists() {
|
||||
return Err(anyhow::anyhow!("No cached models found"));
|
||||
}
|
||||
|
||||
let mut loaded_count = 0;
|
||||
for entry in fs::read_dir(current_dir)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
|
||||
if path.extension().and_then(|s| s.to_str()) == Some("onnx") {
|
||||
if let Some(model_name) = path.file_stem().and_then(|s| s.to_str()) {
|
||||
match self.load_single_cached_model(model_name, &path).await {
|
||||
Ok(_) => {
|
||||
loaded_count += 1;
|
||||
info!("Loaded cached model: {}", model_name);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to load cached model {}: {}", model_name, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info!("Loaded {} cached models from disk", loaded_count);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Load a single cached model from disk
|
||||
async fn load_single_cached_model(&self, model_name: &str, file_path: &Path) -> Result<()> {
|
||||
let file = File::open(file_path)?;
|
||||
let metadata = file.metadata()?;
|
||||
|
||||
// Create memory mapping (will be done in memory_map_models)
|
||||
let mmap = unsafe { MmapOptions::new().map(&file)? };
|
||||
|
||||
let cached_model = CachedModel {
|
||||
model_type: self.parse_model_type(model_name)?,
|
||||
version: semver::Version::new(1, 0, 0), // Default version for cached models
|
||||
file_path: file_path.to_path_buf(),
|
||||
mmap_region: mmap,
|
||||
last_used: Instant::now(),
|
||||
checksum: self.calculate_checksum(file_path)?,
|
||||
priority: self.determine_model_priority(model_name),
|
||||
file_size: metadata.len(),
|
||||
load_time: SystemTime::now(),
|
||||
};
|
||||
|
||||
let mut models = self.cached_models.write().await;
|
||||
models.insert(model_name.to_string(), cached_model);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Parse model type from name
|
||||
fn parse_model_type(&self, model_name: &str) -> Result<ModelType> {
|
||||
match model_name.to_lowercase().as_str() {
|
||||
name if name.contains("tlob") => Ok(ModelType::TlobTransformer),
|
||||
name if name.contains("dqn") => Ok(ModelType::Dqn),
|
||||
name if name.contains("mamba") => Ok(ModelType::Mamba2),
|
||||
name if name.contains("tft") => Ok(ModelType::Tft),
|
||||
name if name.contains("ppo") => Ok(ModelType::Ppo),
|
||||
name if name.contains("liquid") => Ok(ModelType::Liquid),
|
||||
name if name.contains("ensemble") => Ok(ModelType::Ensemble),
|
||||
_ => Err(anyhow::anyhow!("Unknown model type: {}", model_name)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Determine model priority based on type
|
||||
fn determine_model_priority(&self, model_name: &str) -> ModelPriority {
|
||||
match model_name.to_lowercase().as_str() {
|
||||
name if name.contains("tlob") || name.contains("dqn") => ModelPriority::Critical,
|
||||
name if name.contains("mamba") || name.contains("tft") => ModelPriority::Normal,
|
||||
_ => ModelPriority::Low,
|
||||
}
|
||||
}
|
||||
|
||||
/// Memory-map all models for <50μs access
|
||||
async fn memory_map_models(&self) -> Result<()> {
|
||||
let models = self.cached_models.read().await;
|
||||
|
||||
info!("Memory-mapping {} models for high-performance access", models.len());
|
||||
|
||||
// Models are already memory-mapped during loading
|
||||
// This function validates all mappings are working
|
||||
for (name, model) in models.iter() {
|
||||
if model.mmap_region.is_empty() {
|
||||
warn!("Empty memory mapping for model: {}", name);
|
||||
} else {
|
||||
debug!("Model {} mapped: {} bytes", name, model.mmap_region.len());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get model data with <50μs access time
|
||||
pub async fn get_model(&self, model_name: &str) -> Result<&[u8]> {
|
||||
let models = self.cached_models.read().await;
|
||||
|
||||
match models.get(model_name) {
|
||||
Some(model) => {
|
||||
// Zero-copy access to memory-mapped region
|
||||
Ok(model.mmap_region.as_ref())
|
||||
}
|
||||
None => Err(anyhow::anyhow!("Model not cached: {}", model_name)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Start background update monitoring
|
||||
async fn start_background_updates(&self) {
|
||||
let cache = self.clone();
|
||||
let mut interval = interval(Duration::from_secs(cache.config.update_interval_secs));
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
interval.tick().await;
|
||||
|
||||
if let Err(e) = cache.sync_from_s3().await {
|
||||
warn!("Background S3 sync failed: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
info!("Started background model updates every {}s", self.config.update_interval_secs);
|
||||
}
|
||||
|
||||
/// Check if cache is initialized
|
||||
pub async fn is_initialized(&self) -> bool {
|
||||
*self.is_initialized.read().await
|
||||
}
|
||||
|
||||
/// Get cache statistics
|
||||
pub async fn get_cache_stats(&self) -> HashMap<String, serde_json::Value> {
|
||||
let models = self.cached_models.read().await;
|
||||
let mut stats = HashMap::new();
|
||||
|
||||
stats.insert("total_models".to_string(), serde_json::Value::from(models.len()));
|
||||
|
||||
let total_size: u64 = models.values().map(|m| m.file_size).sum();
|
||||
stats.insert("total_size_mb".to_string(), serde_json::Value::from(total_size / (1024 * 1024)));
|
||||
|
||||
let critical_models: usize = models.values()
|
||||
.filter(|m| m.priority == ModelPriority::Critical)
|
||||
.count();
|
||||
stats.insert("critical_models".to_string(), serde_json::Value::from(critical_models));
|
||||
|
||||
stats
|
||||
}
|
||||
}
|
||||
|
||||
impl Clone for ModelCache {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
config: self.config.clone(),
|
||||
cached_models: Arc::clone(&self.cached_models),
|
||||
s3_client: Arc::clone(&self.s3_client),
|
||||
update_broadcaster: self.update_broadcaster.clone(),
|
||||
is_initialized: Arc::clone(&self.is_initialized),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Error types for model cache operations
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ModelCacheError {
|
||||
#[error("Model not found: {0}")]
|
||||
ModelNotFound(String),
|
||||
#[error("S3 error: {0}")]
|
||||
S3Error(String),
|
||||
#[error("File system error: {0}")]
|
||||
FileSystemError(String),
|
||||
#[error("Memory mapping error: {0}")]
|
||||
MemoryMappingError(String),
|
||||
#[error("Configuration error: {0}")]
|
||||
ConfigError(String),
|
||||
}
|
||||
@@ -62,6 +62,9 @@ pub struct TradingServiceState {
|
||||
|
||||
/// Kill switch system for emergency shutdown
|
||||
pub kill_switch_system: Option<Arc<crate::kill_switch_integration::TradingServiceKillSwitch>>,
|
||||
|
||||
/// High-performance model cache for <50μs inference
|
||||
pub model_cache: Option<Arc<crate::model_cache::ModelCache>>,
|
||||
}
|
||||
|
||||
impl TradingServiceState {
|
||||
@@ -72,6 +75,7 @@ impl TradingServiceState {
|
||||
risk_repository: Arc<dyn RiskRepository>,
|
||||
config_repository: Arc<dyn ConfigRepository>,
|
||||
kill_switch_system: Option<Arc<crate::kill_switch_integration::TradingServiceKillSwitch>>,
|
||||
model_cache: Option<Arc<crate::model_cache::ModelCache>>,
|
||||
) -> TradingServiceResult<Self> {
|
||||
// Initialize business logic components (no database coupling)
|
||||
let risk_engine = Arc::new(RwLock::new(RiskEngine::new()));
|
||||
@@ -97,6 +101,7 @@ impl TradingServiceState {
|
||||
event_publisher,
|
||||
metrics,
|
||||
kill_switch_system,
|
||||
model_cache,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -40,10 +40,8 @@ tracing = { workspace = true }
|
||||
rustc-hash = { workspace = true }
|
||||
indexmap = { workspace = true }
|
||||
|
||||
# AWS S3 SDK (optional) - aligned with core crate versions
|
||||
aws-config = { version = "1.5", features = ["behavior-version-latest"], optional = true }
|
||||
aws-sdk-s3 = { version = "1.34", features = ["behavior-version-latest"], optional = true }
|
||||
aws-types = { version = "1.3", optional = true }
|
||||
# Apache Arrow object_store for S3 operations (superior to AWS SDK)
|
||||
object_store = { version = "0.10", features = ["aws"], optional = true }
|
||||
|
||||
# Vault integration removed - use config crate instead
|
||||
|
||||
@@ -72,7 +70,7 @@ wiremock = { workspace = true }
|
||||
default = ["s3"]
|
||||
|
||||
# S3 storage backend
|
||||
s3 = ["aws-config", "aws-sdk-s3", "aws-types"]
|
||||
s3 = ["object_store"]
|
||||
|
||||
# Local file operations only (no cloud dependencies)
|
||||
local-only = []
|
||||
@@ -25,6 +25,7 @@ pub mod error;
|
||||
pub mod config;
|
||||
pub mod metrics;
|
||||
pub mod models;
|
||||
pub mod object_store_backend;
|
||||
|
||||
// Re-export commonly used types and traits
|
||||
pub use error::{StorageError, StorageResult};
|
||||
@@ -37,6 +38,7 @@ pub use s3::{S3Storage, S3StorageConfig, ArchivalDataType, ArchivalMetadata, Arc
|
||||
|
||||
pub use local::{LocalStorage, LocalStorageConfig, FileOperation};
|
||||
pub use models::{ModelStorage, ModelCheckpoint, ModelStorageConfig, ArchivalDataType as ModelDataType, ModelStorageStats, ModelLoader};
|
||||
pub use object_store_backend::ObjectStoreBackend;
|
||||
|
||||
/// Common storage trait for abstracting different storage backends
|
||||
#[async_trait::async_trait]
|
||||
|
||||
338
storage/src/object_store_backend.rs
Normal file
338
storage/src/object_store_backend.rs
Normal file
@@ -0,0 +1,338 @@
|
||||
//! Clean S3 Backend using object_store
|
||||
//!
|
||||
//! Simple, focused implementation of storage traits using object_store crate.
|
||||
//! Integrates with existing config system for credentials and settings.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use object_store::aws::AmazonS3Builder;
|
||||
use object_store::{ObjectStore, Path};
|
||||
use tracing::{debug, info, warn};
|
||||
use futures::TryStreamExt;
|
||||
|
||||
use crate::{Storage, StorageError, StorageMetadata, StorageResult, S3StorageConfig};
|
||||
use config::ConfigManager;
|
||||
|
||||
/// Clean S3 backend using object_store
|
||||
#[derive(Debug)]
|
||||
pub struct ObjectStoreBackend {
|
||||
/// object_store S3 client
|
||||
store: Arc<dyn ObjectStore>,
|
||||
/// Bucket name for operations
|
||||
bucket: String,
|
||||
/// Configuration
|
||||
config: S3StorageConfig,
|
||||
}
|
||||
|
||||
impl ObjectStoreBackend {
|
||||
/// Create new ObjectStoreBackend from config
|
||||
pub async fn new(config: S3StorageConfig, config_manager: ConfigManager) -> StorageResult<Self> {
|
||||
info!(
|
||||
"Initializing ObjectStore S3 backend: bucket={}, region={}",
|
||||
config.bucket_name, config.region
|
||||
);
|
||||
|
||||
// Get AWS credentials from config manager
|
||||
let aws_creds = config_manager
|
||||
.get_aws_credentials(&config.vault_credentials_path)
|
||||
.await
|
||||
.map_err(|e| StorageError::ConfigError {
|
||||
message: format!("Failed to get AWS credentials from config: {}", e),
|
||||
})?;
|
||||
|
||||
// Build the S3 store
|
||||
let store = AmazonS3Builder::new()
|
||||
.with_bucket_name(&config.bucket_name)
|
||||
.with_region(&config.region)
|
||||
.with_access_key_id(&aws_creds.access_key_id)
|
||||
.with_secret_access_key(&aws_creds.secret_access_key)
|
||||
.with_token(aws_creds.session_token.as_deref().unwrap_or_default())
|
||||
.build()
|
||||
.map_err(|e| StorageError::ConfigError {
|
||||
message: format!("Failed to create S3 client: {}", e),
|
||||
})?;
|
||||
|
||||
info!("Successfully initialized ObjectStore S3 backend");
|
||||
|
||||
Ok(Self {
|
||||
store: Arc::new(store),
|
||||
bucket: config.bucket_name.clone(),
|
||||
config,
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert string path to object_store Path
|
||||
fn to_path(&self, path: &str) -> Path {
|
||||
Path::from(path)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Storage for ObjectStoreBackend {
|
||||
async fn store(&self, path: &str, data: &[u8]) -> StorageResult<()> {
|
||||
debug!("Storing {} bytes to S3 path: {}", data.len(), path);
|
||||
|
||||
let s3_path = self.to_path(path);
|
||||
let bytes = Bytes::from(data.to_vec());
|
||||
|
||||
self.store
|
||||
.put(&s3_path, bytes.into())
|
||||
.await
|
||||
.map_err(|e| StorageError::OperationFailed {
|
||||
operation: "put".to_string(),
|
||||
path: path.to_string(),
|
||||
source: Box::new(e),
|
||||
})?;
|
||||
|
||||
debug!("Successfully stored {} bytes to S3: {}", data.len(), path);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn retrieve(&self, path: &str) -> StorageResult<Vec<u8>> {
|
||||
debug!("Retrieving data from S3 path: {}", path);
|
||||
|
||||
let s3_path = self.to_path(path);
|
||||
|
||||
let response = self.store.get(&s3_path).await.map_err(|e| {
|
||||
StorageError::OperationFailed {
|
||||
operation: "get".to_string(),
|
||||
path: path.to_string(),
|
||||
source: Box::new(e),
|
||||
}
|
||||
})?;
|
||||
|
||||
let bytes = response.bytes().await.map_err(|e| {
|
||||
StorageError::OperationFailed {
|
||||
operation: "read_bytes".to_string(),
|
||||
path: path.to_string(),
|
||||
source: Box::new(e),
|
||||
}
|
||||
})?;
|
||||
|
||||
let data = bytes.to_vec();
|
||||
debug!("Successfully retrieved {} bytes from S3: {}", data.len(), path);
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
async fn exists(&self, path: &str) -> StorageResult<bool> {
|
||||
let s3_path = self.to_path(path);
|
||||
|
||||
match self.store.head(&s3_path).await {
|
||||
Ok(_) => Ok(true),
|
||||
Err(object_store::Error::NotFound { .. }) => Ok(false),
|
||||
Err(e) => Err(StorageError::OperationFailed {
|
||||
operation: "head".to_string(),
|
||||
path: path.to_string(),
|
||||
source: Box::new(e),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete(&self, path: &str) -> StorageResult<bool> {
|
||||
debug!("Deleting S3 path: {}", path);
|
||||
|
||||
let s3_path = self.to_path(path);
|
||||
|
||||
match self.store.delete(&s3_path).await {
|
||||
Ok(_) => {
|
||||
debug!("Successfully deleted S3 path: {}", path);
|
||||
Ok(true)
|
||||
}
|
||||
Err(object_store::Error::NotFound { .. }) => {
|
||||
debug!("Path not found for deletion: {}", path);
|
||||
Ok(false)
|
||||
}
|
||||
Err(e) => Err(StorageError::OperationFailed {
|
||||
operation: "delete".to_string(),
|
||||
path: path.to_string(),
|
||||
source: Box::new(e),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn list(&self, prefix: &str) -> StorageResult<Vec<String>> {
|
||||
debug!("Listing S3 objects with prefix: {}", prefix);
|
||||
|
||||
let s3_prefix = if prefix.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(self.to_path(prefix))
|
||||
};
|
||||
|
||||
let list_stream = self.store.list(s3_prefix.as_ref());
|
||||
let objects: Result<Vec<_>, _> = list_stream.try_collect().await;
|
||||
|
||||
let objects = objects.map_err(|e| StorageError::OperationFailed {
|
||||
operation: "list".to_string(),
|
||||
path: prefix.to_string(),
|
||||
source: Box::new(e),
|
||||
})?;
|
||||
|
||||
let paths: Vec<String> = objects.into_iter().map(|meta| meta.location.to_string()).collect();
|
||||
|
||||
debug!("Listed {} objects with prefix: {}", paths.len(), prefix);
|
||||
Ok(paths)
|
||||
}
|
||||
|
||||
async fn metadata(&self, path: &str) -> StorageResult<StorageMetadata> {
|
||||
debug!("Getting metadata for S3 path: {}", path);
|
||||
|
||||
let s3_path = self.to_path(path);
|
||||
|
||||
let meta = self.store.head(&s3_path).await.map_err(|e| {
|
||||
StorageError::OperationFailed {
|
||||
operation: "head".to_string(),
|
||||
path: path.to_string(),
|
||||
source: Box::new(e),
|
||||
}
|
||||
})?;
|
||||
|
||||
Ok(StorageMetadata {
|
||||
path: path.to_string(),
|
||||
size: meta.size,
|
||||
content_type: None, // object_store doesn't expose content-type in head
|
||||
last_modified: meta.last_modified,
|
||||
etag: meta.e_tag.clone(),
|
||||
tags: std::collections::HashMap::new(), // Tags would need separate API call
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Import CheckpointMetadata from ml crate for checkpoint storage implementation
|
||||
use ml::checkpoint::{CheckpointMetadata, CheckpointStorage};
|
||||
use ml::MLError;
|
||||
|
||||
#[async_trait]
|
||||
impl CheckpointStorage for ObjectStoreBackend {
|
||||
async fn save_checkpoint(
|
||||
&self,
|
||||
filename: &str,
|
||||
data: &[u8],
|
||||
metadata: &CheckpointMetadata,
|
||||
) -> Result<(), MLError> {
|
||||
// Store checkpoint data
|
||||
let checkpoint_path = format!("checkpoints/{}", filename);
|
||||
self.store(&checkpoint_path, data)
|
||||
.await
|
||||
.map_err(|e| MLError::ModelError(format!("Failed to save checkpoint: {}", e)))?;
|
||||
|
||||
// Store metadata as separate JSON object
|
||||
let metadata_path = format!("checkpoints/metadata/{}.json", filename);
|
||||
let metadata_json = serde_json::to_vec_pretty(metadata)
|
||||
.map_err(|e| MLError::ModelError(format!("Failed to serialize metadata: {}", e)))?;
|
||||
|
||||
self.store(&metadata_path, &metadata_json)
|
||||
.await
|
||||
.map_err(|e| MLError::ModelError(format!("Failed to save metadata: {}", e)))?;
|
||||
|
||||
info!(
|
||||
"Successfully saved checkpoint {} ({} bytes)",
|
||||
filename,
|
||||
data.len()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn load_checkpoint(&self, filename: &str) -> Result<Vec<u8>, MLError> {
|
||||
let checkpoint_path = format!("checkpoints/{}", filename);
|
||||
|
||||
let data = self
|
||||
.retrieve(&checkpoint_path)
|
||||
.await
|
||||
.map_err(|e| MLError::ModelError(format!("Failed to load checkpoint: {}", e)))?;
|
||||
|
||||
debug!("Loaded checkpoint {} ({} bytes)", filename, data.len());
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
async fn delete_checkpoint(&self, filename: &str) -> Result<(), MLError> {
|
||||
let checkpoint_path = format!("checkpoints/{}", filename);
|
||||
let metadata_path = format!("checkpoints/metadata/{}.json", filename);
|
||||
|
||||
// Delete both checkpoint and metadata
|
||||
let _checkpoint_deleted = self.delete(&checkpoint_path).await.unwrap_or(false);
|
||||
let _metadata_deleted = self.delete(&metadata_path).await.unwrap_or(false);
|
||||
|
||||
info!("Deleted checkpoint {}", filename);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_all_checkpoints(&self) -> Result<Vec<CheckpointMetadata>, MLError> {
|
||||
let metadata_prefix = "checkpoints/metadata/";
|
||||
let paths = self
|
||||
.list(metadata_prefix)
|
||||
.await
|
||||
.map_err(|e| MLError::ModelError(format!("Failed to list checkpoints: {}", e)))?;
|
||||
|
||||
let mut checkpoints = Vec::new();
|
||||
|
||||
for path in paths {
|
||||
if path.ends_with(".json") {
|
||||
match self.retrieve(&path).await {
|
||||
Ok(metadata_json) => {
|
||||
match serde_json::from_slice::<CheckpointMetadata>(&metadata_json) {
|
||||
Ok(metadata) => checkpoints.push(metadata),
|
||||
Err(e) => {
|
||||
warn!("Failed to parse checkpoint metadata {}: {}", path, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to load checkpoint metadata {}: {}", path, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
debug!("Listed {} checkpoints", checkpoints.len());
|
||||
Ok(checkpoints)
|
||||
}
|
||||
|
||||
async fn checkpoint_exists(&self, filename: &str) -> bool {
|
||||
let checkpoint_path = format!("checkpoints/{}", filename);
|
||||
self.exists(&checkpoint_path).await.unwrap_or(false)
|
||||
}
|
||||
|
||||
async fn get_storage_stats(&self) -> Result<ml::checkpoint::storage::StorageStats, MLError> {
|
||||
let checkpoints = self.list_all_checkpoints().await?;
|
||||
let total_checkpoints = checkpoints.len() as u64;
|
||||
|
||||
let mut total_bytes = 0u64;
|
||||
for metadata in &checkpoints {
|
||||
total_bytes += metadata.file_size;
|
||||
}
|
||||
|
||||
let avg_checkpoint_size = if total_checkpoints > 0 {
|
||||
total_bytes / total_checkpoints
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
Ok(ml::checkpoint::storage::StorageStats {
|
||||
total_checkpoints,
|
||||
total_bytes,
|
||||
available_bytes: u64::MAX, // S3 has virtually unlimited storage
|
||||
avg_checkpoint_size,
|
||||
backend_type: format!("object_store_s3:{}", self.bucket),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_path_conversion() {
|
||||
let backend = ObjectStoreBackend {
|
||||
store: Arc::new(object_store::memory::InMemory::new()),
|
||||
bucket: "test".to_string(),
|
||||
config: S3StorageConfig::default(),
|
||||
};
|
||||
|
||||
let path = backend.to_path("test/path");
|
||||
assert_eq!(path.as_ref(), "test/path");
|
||||
}
|
||||
}
|
||||
@@ -6,16 +6,18 @@
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use std::sync::Arc;
|
||||
use async_trait::async_trait;
|
||||
use aws_config::BehaviorVersion;
|
||||
use aws_sdk_s3::{Client as S3Client, Config as S3Config};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{ServerSideEncryption, StorageClass, Tagging, Tag};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::{debug, info, warn, error};
|
||||
use uuid::Uuid;
|
||||
|
||||
use object_store::aws::AmazonS3;
|
||||
use object_store::{ObjectStore, path::Path, PutPayload, GetOptions, ListResult};
|
||||
use object_store::aws::AmazonS3Builder;
|
||||
use bytes::Bytes;
|
||||
|
||||
use crate::{Storage, StorageError, StorageMetadata, StorageResult};
|
||||
use config::{ConfigManager, ConfigCategory};
|
||||
|
||||
@@ -145,16 +147,16 @@ impl ArchivalDataType {
|
||||
}
|
||||
}
|
||||
|
||||
/// Get appropriate storage class for data type
|
||||
pub fn storage_class(&self) -> StorageClass {
|
||||
/// Get appropriate storage class name for data type (for metadata/logging)
|
||||
pub fn storage_class_name(&self) -> &'static str {
|
||||
match self {
|
||||
ArchivalDataType::MarketData => StorageClass::StandardIa,
|
||||
ArchivalDataType::TradingLogs => StorageClass::Standard, // Frequently accessed
|
||||
ArchivalDataType::RiskLogs => StorageClass::Standard,
|
||||
ArchivalDataType::ModelCheckpoints => StorageClass::StandardIa,
|
||||
ArchivalDataType::PerformanceMetrics => StorageClass::StandardIa,
|
||||
ArchivalDataType::ComplianceData => StorageClass::Glacier, // Long-term retention
|
||||
ArchivalDataType::ApplicationData => StorageClass::Standard,
|
||||
ArchivalDataType::MarketData => "STANDARD_IA",
|
||||
ArchivalDataType::TradingLogs => "STANDARD", // Frequently accessed
|
||||
ArchivalDataType::RiskLogs => "STANDARD",
|
||||
ArchivalDataType::ModelCheckpoints => "STANDARD_IA",
|
||||
ArchivalDataType::PerformanceMetrics => "STANDARD_IA",
|
||||
ArchivalDataType::ComplianceData => "GLACIER", // Long-term retention
|
||||
ArchivalDataType::ApplicationData => "STANDARD",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -204,7 +206,7 @@ pub struct ArchivalStats {
|
||||
|
||||
/// S3 storage implementation with config integration
|
||||
pub struct S3Storage {
|
||||
client: S3Client,
|
||||
client: Arc<AmazonS3>,
|
||||
config: S3StorageConfig,
|
||||
config_manager: ConfigManager,
|
||||
}
|
||||
@@ -240,45 +242,47 @@ impl S3Storage {
|
||||
message: "AWS secret key not found in configuration".to_string(),
|
||||
})?;
|
||||
|
||||
// Create AWS credentials from config values
|
||||
let aws_credentials = aws_sdk_s3::config::Credentials::new(
|
||||
access_key,
|
||||
secret_key,
|
||||
None, // session_token
|
||||
None, // expiry
|
||||
"foxhunt-config-crate",
|
||||
);
|
||||
|
||||
let aws_config = aws_config::defaults(BehaviorVersion::latest())
|
||||
.region(&config.region)
|
||||
.credentials_provider(aws_credentials)
|
||||
.load()
|
||||
.await;
|
||||
|
||||
let client = S3Client::new(&aws_config);
|
||||
// Test connection and bucket access
|
||||
client
|
||||
.head_bucket()
|
||||
.bucket(&config.bucket_name)
|
||||
.send()
|
||||
.await
|
||||
// Create AmazonS3 client using object_store - much simpler!
|
||||
let client = AmazonS3Builder::new()
|
||||
.with_bucket_name(&config.bucket_name)
|
||||
.with_region(&config.region)
|
||||
.with_access_key_id(&access_key)
|
||||
.with_secret_access_key(&secret_key)
|
||||
.build()
|
||||
.map_err(|e| StorageError::S3Error {
|
||||
message: format!("Failed to connect to S3 bucket '{}': {}", config.bucket_name, e),
|
||||
message: format!("Failed to create S3 client: {}", e),
|
||||
})?;
|
||||
|
||||
|
||||
// Test connection with a simple head operation
|
||||
let test_path = Path::from("_connection_test");
|
||||
match client.head(&test_path).await {
|
||||
Ok(_) => {
|
||||
// Object exists (unlikely but OK)
|
||||
},
|
||||
Err(object_store::Error::NotFound { .. }) => {
|
||||
// Expected - bucket is accessible but object doesn't exist
|
||||
},
|
||||
Err(e) => {
|
||||
return Err(StorageError::S3Error {
|
||||
message: format!("Failed to connect to S3 bucket '{}': {}", config.bucket_name, e),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
info!("Successfully connected to S3 bucket: {}", config.bucket_name);
|
||||
|
||||
|
||||
let storage = Self {
|
||||
client,
|
||||
client: Arc::new(client),
|
||||
config,
|
||||
config_manager,
|
||||
};
|
||||
|
||||
// Setup lifecycle policies if enabled
|
||||
|
||||
// Note: lifecycle policies need to be managed separately with object_store
|
||||
// This is typically done via AWS CLI or infrastructure-as-code
|
||||
if storage.config.enable_lifecycle {
|
||||
storage.setup_lifecycle_policies().await?;
|
||||
warn!("Lifecycle policies must be configured via AWS CLI or infrastructure - object_store doesn't support lifecycle management directly");
|
||||
}
|
||||
|
||||
|
||||
Ok(storage)
|
||||
}
|
||||
|
||||
@@ -359,147 +363,41 @@ impl S3Storage {
|
||||
format!("{:x}", hasher.finalize())
|
||||
}
|
||||
|
||||
/// Create S3 object tags from metadata
|
||||
fn create_object_tags(&self, metadata: &HashMap<String, String>) -> Vec<Tag> {
|
||||
let mut tags = Vec::new();
|
||||
|
||||
tags.push(Tag::builder().key("service").value("foxhunt-hft").build().unwrap());
|
||||
tags.push(Tag::builder().key("managed_by").value("storage-crate").build().unwrap());
|
||||
|
||||
// Add custom tags
|
||||
for (key, value) in metadata {
|
||||
tags.push(Tag::builder().key(key).value(value).build().unwrap());
|
||||
/// Create object metadata including service tags
|
||||
fn create_object_metadata(&self, base_metadata: &HashMap<String, String>) -> HashMap<String, String> {
|
||||
let mut metadata = HashMap::new();
|
||||
|
||||
// Add service tags
|
||||
metadata.insert("service".to_string(), "foxhunt-hft".to_string());
|
||||
metadata.insert("managed_by".to_string(), "storage-crate".to_string());
|
||||
|
||||
// Add custom metadata
|
||||
for (key, value) in base_metadata {
|
||||
metadata.insert(key.clone(), value.clone());
|
||||
}
|
||||
|
||||
tags
|
||||
|
||||
metadata
|
||||
}
|
||||
|
||||
/// Setup S3 lifecycle policies for cost optimization
|
||||
async fn setup_lifecycle_policies(&self) -> StorageResult<()> {
|
||||
use aws_sdk_s3::types::{
|
||||
LifecycleConfiguration, LifecycleRule, LifecycleRuleStatus,
|
||||
Transition, LifecycleExpiration
|
||||
};
|
||||
|
||||
info!("Setting up S3 lifecycle policies for bucket: {}", self.config.bucket_name);
|
||||
|
||||
let rules = vec![
|
||||
// Transition market data to Glacier after 90 days
|
||||
LifecycleRule::builder()
|
||||
.id("market-data-transition")
|
||||
.status(LifecycleRuleStatus::Enabled)
|
||||
.filter(aws_sdk_s3::types::LifecycleRuleFilter::Prefix("market-data/".to_string()))
|
||||
.transitions(
|
||||
Transition::builder()
|
||||
.days(90)
|
||||
.storage_class(StorageClass::Glacier)
|
||||
.build()
|
||||
.unwrap()
|
||||
)
|
||||
.build()
|
||||
.unwrap(),
|
||||
|
||||
// Transition model checkpoints to Standard-IA after 30 days, Glacier after 180 days
|
||||
LifecycleRule::builder()
|
||||
.id("model-checkpoints-transition")
|
||||
.status(LifecycleRuleStatus::Enabled)
|
||||
.filter(aws_sdk_s3::types::LifecycleRuleFilter::Prefix("model-checkpoints/".to_string()))
|
||||
.transitions(
|
||||
Transition::builder()
|
||||
.days(30)
|
||||
.storage_class(StorageClass::StandardIa)
|
||||
.build()
|
||||
.unwrap()
|
||||
)
|
||||
.transitions(
|
||||
Transition::builder()
|
||||
.days(180)
|
||||
.storage_class(StorageClass::Glacier)
|
||||
.build()
|
||||
.unwrap()
|
||||
)
|
||||
.build()
|
||||
.unwrap(),
|
||||
|
||||
// Delete performance metrics after retention period
|
||||
LifecycleRule::builder()
|
||||
.id("data-retention-policy")
|
||||
.status(LifecycleRuleStatus::Enabled)
|
||||
.filter(aws_sdk_s3::types::LifecycleRuleFilter::And(
|
||||
aws_sdk_s3::types::LifecycleRuleAndOperator::builder()
|
||||
.prefix("performance-metrics/")
|
||||
.build()
|
||||
.unwrap()
|
||||
))
|
||||
.expiration(
|
||||
LifecycleExpiration::builder()
|
||||
.days(self.config.retention_days as i32)
|
||||
.build()
|
||||
.unwrap()
|
||||
)
|
||||
.build()
|
||||
.unwrap(),
|
||||
];
|
||||
|
||||
let lifecycle_config = LifecycleConfiguration::builder()
|
||||
.set_rules(Some(rules))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
self.client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(&self.config.bucket_name)
|
||||
.lifecycle_configuration(lifecycle_config)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| StorageError::S3Error {
|
||||
message: format!("Failed to set up S3 lifecycle policies: {}", e),
|
||||
})?;
|
||||
|
||||
info!("Successfully configured S3 lifecycle policies");
|
||||
Ok(())
|
||||
/// Note: S3 lifecycle policies must be configured separately via AWS CLI or infrastructure
|
||||
/// object_store doesn't provide direct lifecycle policy management
|
||||
async fn log_lifecycle_policy_info(&self) {
|
||||
if self.config.enable_lifecycle {
|
||||
info!("Lifecycle policies enabled in config but must be managed via AWS CLI/CloudFormation");
|
||||
info!("Suggested lifecycle policy setup for bucket '{}'", self.config.bucket_name);
|
||||
info!(" - Transition market-data/ to Glacier after 90 days");
|
||||
info!(" - Transition model-checkpoints/ to Standard-IA after 30 days, Glacier after 180 days");
|
||||
info!(" - Delete performance-metrics/ after {} days", self.config.retention_days);
|
||||
info!("Configure these via AWS CLI: aws s3api put-bucket-lifecycle-configuration");
|
||||
}
|
||||
}
|
||||
|
||||
/// Refresh AWS credentials from config system if needed
|
||||
async fn ensure_credentials_valid(&mut self) -> StorageResult<()> {
|
||||
// Get fresh credentials from config system
|
||||
let access_key = self.config_manager
|
||||
.get_string(ConfigCategory::Environment, "aws_access_key_id")
|
||||
.await
|
||||
.map_err(|e| StorageError::AuthError {
|
||||
message: format!("Failed to refresh AWS access key from config: {}", e),
|
||||
})?
|
||||
.ok_or_else(|| StorageError::AuthError {
|
||||
message: "AWS access key not found in configuration".to_string(),
|
||||
})?;
|
||||
|
||||
let secret_key = self.config_manager
|
||||
.get_string(ConfigCategory::Environment, "aws_secret_access_key")
|
||||
.await
|
||||
.map_err(|e| StorageError::AuthError {
|
||||
message: format!("Failed to refresh AWS secret key from config: {}", e),
|
||||
})?
|
||||
.ok_or_else(|| StorageError::AuthError {
|
||||
message: "AWS secret key not found in configuration".to_string(),
|
||||
})?;
|
||||
|
||||
debug!("Refreshing AWS credentials from config system");
|
||||
|
||||
let aws_credentials = aws_sdk_s3::config::Credentials::new(
|
||||
access_key,
|
||||
secret_key,
|
||||
None, // session_token
|
||||
None, // expiry
|
||||
"foxhunt-config-crate", );
|
||||
let aws_config = aws_config::defaults(BehaviorVersion::latest())
|
||||
.region(&self.config.region)
|
||||
.credentials_provider(aws_credentials)
|
||||
.load()
|
||||
.await;
|
||||
|
||||
self.client = S3Client::new(&aws_config);
|
||||
info!("AWS credentials refreshed from config system");
|
||||
|
||||
/// Note: Credential refresh with object_store requires recreating the client
|
||||
/// This is handled automatically by the object_store library in most cases
|
||||
async fn recreate_client_if_needed(&mut self) -> StorageResult<()> {
|
||||
// With object_store, credential refresh typically requires client recreation
|
||||
// This is usually handled automatically, but we can recreate if needed
|
||||
info!("Note: object_store handles credential refresh internally");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -509,47 +407,40 @@ impl Storage for S3Storage {
|
||||
async fn store(&self, path: &str, data: &[u8]) -> StorageResult<()> {
|
||||
let start = std::time::Instant::now();
|
||||
let s3_key = self.generate_s3_key(path, None);
|
||||
|
||||
|
||||
debug!("Storing {} bytes to S3: {}", data.len(), s3_key);
|
||||
|
||||
|
||||
// Compress data if enabled
|
||||
let (processed_data, compression_type) = self.compress_data(data)?;
|
||||
let checksum = self.calculate_checksum(data);
|
||||
|
||||
|
||||
// Create object metadata
|
||||
let mut object_metadata = HashMap::new();
|
||||
object_metadata.insert("original_size".to_string(), data.len().to_string());
|
||||
object_metadata.insert("checksum".to_string(), checksum);
|
||||
object_metadata.insert("compression".to_string(), compression_type.unwrap_or_else(|| "none".to_string()));
|
||||
object_metadata.insert("stored_at".to_string(), Utc::now().to_rfc3339());
|
||||
|
||||
// Create tags
|
||||
let tags = self.create_object_tags(&object_metadata);
|
||||
let tagging = Tagging::builder().set_tag_set(Some(tags)).build().unwrap();
|
||||
|
||||
// Upload to S3 with encryption
|
||||
let body = ByteStream::from(processed_data);
|
||||
|
||||
|
||||
// Convert to object_store Path
|
||||
let object_path = Path::from(s3_key.clone());
|
||||
|
||||
// Create payload with metadata
|
||||
let payload = PutPayload::from_bytes(Bytes::from(processed_data))
|
||||
.with_content_type("application/octet-stream")
|
||||
.with_metadata(object_metadata);
|
||||
|
||||
// Upload to S3 using object_store
|
||||
self.client
|
||||
.put_object()
|
||||
.bucket(&self.config.bucket_name)
|
||||
.key(&s3_key)
|
||||
.body(body)
|
||||
.set_metadata(Some(object_metadata))
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.storage_class(StorageClass::from(self.config.default_storage_class.as_str()))
|
||||
.tagging(tagging)
|
||||
.content_type("application/octet-stream")
|
||||
.send()
|
||||
.put(&object_path, payload)
|
||||
.await
|
||||
.map_err(|e| StorageError::S3Error {
|
||||
message: format!("Failed to store data to S3: {}", e),
|
||||
})?;
|
||||
|
||||
|
||||
let duration = start.elapsed();
|
||||
crate::metrics::get_metrics().record_operation("store", "s3", duration, true);
|
||||
crate::metrics::get_metrics().record_transfer("store", "s3", data.len() as u64, duration);
|
||||
|
||||
|
||||
info!("Successfully stored {} bytes to S3: {}", data.len(), s3_key);
|
||||
Ok(())
|
||||
}
|
||||
@@ -557,38 +448,36 @@ impl Storage for S3Storage {
|
||||
async fn retrieve(&self, path: &str) -> StorageResult<Vec<u8>> {
|
||||
let start = std::time::Instant::now();
|
||||
let s3_key = self.generate_s3_key(path, None);
|
||||
|
||||
|
||||
debug!("Retrieving data from S3: {}", s3_key);
|
||||
|
||||
let response = self.client
|
||||
.get_object()
|
||||
.bucket(&self.config.bucket_name)
|
||||
.key(&s3_key)
|
||||
.send()
|
||||
|
||||
let object_path = Path::from(s3_key.clone());
|
||||
|
||||
let get_result = self.client
|
||||
.get(&object_path)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
if let Some(service_err) = e.as_service_error() {
|
||||
if service_err.is_no_such_key() {
|
||||
return StorageError::NotFound { path: path.to_string() };
|
||||
match e {
|
||||
object_store::Error::NotFound { .. } => {
|
||||
StorageError::NotFound { path: path.to_string() }
|
||||
},
|
||||
_ => StorageError::S3Error {
|
||||
message: format!("Failed to retrieve data from S3: {}", e),
|
||||
}
|
||||
}
|
||||
StorageError::S3Error {
|
||||
message: format!("Failed to retrieve data from S3: {}", e),
|
||||
}
|
||||
})?;
|
||||
|
||||
// Read the body
|
||||
let body = response.body.collect().await
|
||||
|
||||
// Read the response data
|
||||
let compressed_data = get_result.bytes().await
|
||||
.map_err(|e| StorageError::S3Error {
|
||||
message: format!("Failed to read S3 object body: {}", e),
|
||||
})?;
|
||||
let compressed_data = body.into_bytes().to_vec();
|
||||
|
||||
// Check if data was compressed
|
||||
let compression_type = response.metadata
|
||||
.and_then(|m| m.get("compression"))
|
||||
message: format!("Failed to read S3 object data: {}", e),
|
||||
})?.to_vec();
|
||||
|
||||
// Check if data was compressed from metadata
|
||||
let compression_type = get_result.meta.metadata
|
||||
.get("compression")
|
||||
.cloned();
|
||||
|
||||
|
||||
let final_data = if let Some(comp_type) = compression_type {
|
||||
if comp_type == "gzip" {
|
||||
self.decompress_data(&compressed_data)?
|
||||
@@ -602,11 +491,11 @@ impl Storage for S3Storage {
|
||||
} else {
|
||||
compressed_data
|
||||
};
|
||||
|
||||
|
||||
let duration = start.elapsed();
|
||||
crate::metrics::get_metrics().record_operation("retrieve", "s3", duration, true);
|
||||
crate::metrics::get_metrics().record_transfer("retrieve", "s3", final_data.len() as u64, duration);
|
||||
|
||||
|
||||
debug!("Successfully retrieved {} bytes from S3", final_data.len());
|
||||
Ok(final_data)
|
||||
}
|
||||
@@ -614,35 +503,21 @@ impl Storage for S3Storage {
|
||||
async fn exists(&self, path: &str) -> StorageResult<bool> {
|
||||
let start = std::time::Instant::now();
|
||||
let s3_key = self.generate_s3_key(path, None);
|
||||
|
||||
let exists = match self.client
|
||||
.head_object()
|
||||
.bucket(&self.config.bucket_name)
|
||||
.key(&s3_key)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
let object_path = Path::from(s3_key.clone());
|
||||
|
||||
let exists = match self.client.head(&object_path).await {
|
||||
Ok(_) => true,
|
||||
Err(object_store::Error::NotFound { .. }) => false,
|
||||
Err(e) => {
|
||||
if let Some(service_err) = e.as_service_error() {
|
||||
if service_err.is_no_such_key() {
|
||||
false
|
||||
} else {
|
||||
return Err(StorageError::S3Error {
|
||||
message: format!("Failed to check if object exists: {}", e),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
return Err(StorageError::S3Error {
|
||||
message: format!("Failed to check if object exists: {}", e),
|
||||
});
|
||||
}
|
||||
return Err(StorageError::S3Error {
|
||||
message: format!("Failed to check if object exists: {}", e),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
let duration = start.elapsed();
|
||||
crate::metrics::get_metrics().record_operation("exists", "s3", duration, true);
|
||||
|
||||
|
||||
debug!("S3 path {} exists: {}", path, exists);
|
||||
Ok(exists)
|
||||
}
|
||||
@@ -650,29 +525,22 @@ impl Storage for S3Storage {
|
||||
async fn delete(&self, path: &str) -> StorageResult<bool> {
|
||||
let start = std::time::Instant::now();
|
||||
let s3_key = self.generate_s3_key(path, None);
|
||||
|
||||
let object_path = Path::from(s3_key.clone());
|
||||
|
||||
debug!("Deleting S3 object: {}", s3_key);
|
||||
|
||||
match self.client
|
||||
.delete_object()
|
||||
.bucket(&self.config.bucket_name)
|
||||
.key(&s3_key)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
|
||||
match self.client.delete(&object_path).await {
|
||||
Ok(_) => {
|
||||
let duration = start.elapsed();
|
||||
crate::metrics::get_metrics().record_operation("delete", "s3", duration, true);
|
||||
info!("Successfully deleted S3 object: {}", s3_key);
|
||||
Ok(true)
|
||||
}
|
||||
Err(object_store::Error::NotFound { .. }) => {
|
||||
debug!("S3 object not found: {}", s3_key);
|
||||
Ok(false)
|
||||
}
|
||||
Err(e) => {
|
||||
if let Some(service_err) = e.as_service_error() {
|
||||
if service_err.is_no_such_key() {
|
||||
debug!("S3 object not found: {}", s3_key);
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Err(StorageError::S3Error {
|
||||
message: format!("Failed to delete S3 object: {}", e),
|
||||
})
|
||||
@@ -687,48 +555,41 @@ impl Storage for S3Storage {
|
||||
} else {
|
||||
self.generate_s3_key(prefix, None)
|
||||
};
|
||||
|
||||
|
||||
debug!("Listing S3 objects with prefix: {}", s3_prefix);
|
||||
|
||||
|
||||
let mut paths = Vec::new();
|
||||
let mut continuation_token: Option<String> = None;
|
||||
|
||||
loop {
|
||||
let mut request = self.client
|
||||
.list_objects_v2()
|
||||
.bucket(&self.config.bucket_name)
|
||||
.prefix(&s3_prefix);
|
||||
|
||||
if let Some(token) = &continuation_token {
|
||||
request = request.continuation_token(token);
|
||||
}
|
||||
|
||||
let response = request.send().await
|
||||
.map_err(|e| StorageError::S3Error {
|
||||
message: format!("Failed to list S3 objects: {}", e),
|
||||
})?;
|
||||
|
||||
if let Some(contents) = response.contents {
|
||||
for object in contents {
|
||||
if let Some(key) = object.key {
|
||||
// Convert S3 key back to storage path
|
||||
if let Some(relative_path) = key.strip_prefix(&format!("{}/", s3_prefix)) {
|
||||
paths.push(relative_path.to_string());
|
||||
}
|
||||
}
|
||||
let prefix_path = if s3_prefix.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(&Path::from(s3_prefix.clone()))
|
||||
};
|
||||
|
||||
// Use object_store list with stream
|
||||
let mut list_stream = self.client.list(prefix_path).await
|
||||
.map_err(|e| StorageError::S3Error {
|
||||
message: format!("Failed to list S3 objects: {}", e),
|
||||
})?;
|
||||
|
||||
// Collect all results
|
||||
use futures::TryStreamExt;
|
||||
while let Some(meta) = list_stream.try_next().await
|
||||
.map_err(|e| StorageError::S3Error {
|
||||
message: format!("Failed to read S3 list results: {}", e),
|
||||
})?
|
||||
{
|
||||
let key = meta.location.as_ref();
|
||||
// Convert S3 key back to storage path
|
||||
if let Some(relative_path) = key.strip_prefix(&format!("{}/", s3_prefix)) {
|
||||
if !relative_path.is_empty() {
|
||||
paths.push(relative_path.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
if response.is_truncated.unwrap_or(false) {
|
||||
continuation_token = response.next_continuation_token;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
let duration = start.elapsed();
|
||||
crate::metrics::get_metrics().record_operation("list", "s3", duration, true);
|
||||
|
||||
|
||||
debug!("Listed {} S3 objects with prefix '{}'", paths.len(), prefix);
|
||||
Ok(paths)
|
||||
}
|
||||
@@ -736,41 +597,36 @@ impl Storage for S3Storage {
|
||||
async fn metadata(&self, path: &str) -> StorageResult<StorageMetadata> {
|
||||
let start = std::time::Instant::now();
|
||||
let s3_key = self.generate_s3_key(path, None);
|
||||
|
||||
let response = self.client
|
||||
.head_object()
|
||||
.bucket(&self.config.bucket_name)
|
||||
.key(&s3_key)
|
||||
.send()
|
||||
let object_path = Path::from(s3_key.clone());
|
||||
|
||||
let head_result = self.client
|
||||
.head(&object_path)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
if let Some(service_err) = e.as_service_error() {
|
||||
if service_err.is_no_such_key() {
|
||||
return StorageError::NotFound { path: path.to_string() };
|
||||
match e {
|
||||
object_store::Error::NotFound { .. } => {
|
||||
StorageError::NotFound { path: path.to_string() }
|
||||
},
|
||||
_ => StorageError::S3Error {
|
||||
message: format!("Failed to get S3 object metadata: {}", e),
|
||||
}
|
||||
}
|
||||
StorageError::S3Error {
|
||||
message: format!("Failed to get S3 object metadata: {}", e),
|
||||
}
|
||||
})?;
|
||||
|
||||
let size = response.content_length.unwrap_or(0) as u64;
|
||||
let last_modified = response.last_modified
|
||||
.map(|ts| DateTime::from_timestamp(ts.secs(), ts.subsec_nanos()).unwrap_or_else(|| Utc::now()))
|
||||
.unwrap_or_else(|| Utc::now());
|
||||
let etag = response.e_tag;
|
||||
|
||||
let metadata_map = response.metadata.unwrap_or_default();
|
||||
|
||||
let size = head_result.size;
|
||||
let last_modified = head_result.last_modified;
|
||||
let etag = head_result.e_tag.clone();
|
||||
|
||||
let mut tags = HashMap::new();
|
||||
|
||||
// Convert metadata to tags
|
||||
for (key, value) in metadata_map {
|
||||
tags.insert(key, value);
|
||||
|
||||
// Convert object_store metadata to tags
|
||||
for (key, value) in &head_result.metadata {
|
||||
tags.insert(key.clone(), value.clone());
|
||||
}
|
||||
|
||||
|
||||
let duration = start.elapsed();
|
||||
crate::metrics::get_metrics().record_operation("metadata", "s3", duration, true);
|
||||
|
||||
|
||||
Ok(StorageMetadata {
|
||||
path: path.to_string(),
|
||||
size,
|
||||
@@ -828,44 +684,38 @@ impl S3Storage {
|
||||
};
|
||||
|
||||
// Create object metadata for S3
|
||||
let mut object_metadata = HashMap::new();
|
||||
object_metadata.insert("archive_id".to_string(), archive_id.to_string());
|
||||
object_metadata.insert("data_type".to_string(), format!("{:?}", data_type));
|
||||
object_metadata.insert("source_id".to_string(), source_id);
|
||||
object_metadata.insert("original_size".to_string(), data.len().to_string());
|
||||
object_metadata.insert("checksum".to_string(), checksum);
|
||||
let mut base_metadata = HashMap::new();
|
||||
base_metadata.insert("archive_id".to_string(), archive_id.to_string());
|
||||
base_metadata.insert("data_type".to_string(), format!("{:?}", data_type));
|
||||
base_metadata.insert("source_id".to_string(), source_id);
|
||||
base_metadata.insert("original_size".to_string(), data.len().to_string());
|
||||
base_metadata.insert("checksum".to_string(), checksum);
|
||||
base_metadata.insert("storage_class".to_string(), data_type.storage_class_name().to_string());
|
||||
if let Some(compressed_size) = metadata.compressed_size {
|
||||
object_metadata.insert("compressed_size".to_string(), compressed_size.to_string());
|
||||
base_metadata.insert("compressed_size".to_string(), compressed_size.to_string());
|
||||
}
|
||||
if let Some(comp_type) = &compression_type {
|
||||
object_metadata.insert("compression".to_string(), comp_type.clone());
|
||||
base_metadata.insert("compression".to_string(), comp_type.clone());
|
||||
}
|
||||
|
||||
|
||||
// Add custom metadata
|
||||
if let Some(additional) = additional_tags {
|
||||
for (key, value) in additional {
|
||||
object_metadata.insert(key, value);
|
||||
base_metadata.insert(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
// Create tags
|
||||
let tags = self.create_object_tags(&object_metadata);
|
||||
let tagging = Tagging::builder().set_tag_set(Some(tags)).build().unwrap();
|
||||
|
||||
// Upload to S3
|
||||
let body = ByteStream::from(processed_data);
|
||||
|
||||
// Create final metadata with service tags
|
||||
let object_metadata = self.create_object_metadata(&base_metadata);
|
||||
|
||||
// Convert to object_store Path and upload
|
||||
let object_path = Path::from(s3_key.clone());
|
||||
let payload = PutPayload::from_bytes(Bytes::from(processed_data))
|
||||
.with_content_type("application/octet-stream")
|
||||
.with_metadata(object_metadata);
|
||||
|
||||
self.client
|
||||
.put_object()
|
||||
.bucket(&self.config.bucket_name)
|
||||
.key(&s3_key)
|
||||
.body(body)
|
||||
.set_metadata(Some(object_metadata))
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.storage_class(data_type.storage_class())
|
||||
.tagging(tagging)
|
||||
.content_type("application/octet-stream")
|
||||
.send()
|
||||
.put(&object_path, payload)
|
||||
.await
|
||||
.map_err(|e| StorageError::S3Error {
|
||||
message: format!("Failed to archive data to S3: {}", e),
|
||||
@@ -884,57 +734,54 @@ impl S3Storage {
|
||||
/// Get archival statistics
|
||||
pub async fn get_archival_stats(&self) -> StorageResult<ArchivalStats> {
|
||||
info!("Calculating archival statistics for bucket: {}", self.config.bucket_name);
|
||||
|
||||
|
||||
let mut total_objects = 0u64;
|
||||
let mut total_size = 0u64;
|
||||
let mut storage_by_type = HashMap::new();
|
||||
let mut compression_ratios = Vec::new();
|
||||
|
||||
let mut continuation_token: Option<String> = None;
|
||||
|
||||
loop {
|
||||
let mut request = self.client
|
||||
.list_objects_v2()
|
||||
.bucket(&self.config.bucket_name);
|
||||
|
||||
if let Some(token) = &continuation_token {
|
||||
request = request.continuation_token(token);
|
||||
}
|
||||
|
||||
let response = request.send().await
|
||||
.map_err(|e| StorageError::S3Error {
|
||||
message: format!("Failed to list objects for statistics: {}", e),
|
||||
})?;
|
||||
|
||||
if let Some(contents) = response.contents {
|
||||
for object in contents {
|
||||
if let (Some(key), Some(size)) = (object.key, object.size) {
|
||||
total_objects += 1;
|
||||
total_size += size as u64;
|
||||
|
||||
// Categorize by data type
|
||||
let data_type = key.split('/').next().unwrap_or("unknown").to_string();
|
||||
*storage_by_type.entry(data_type).or_insert(0) += size as u64;
|
||||
}
|
||||
|
||||
// List all objects using object_store
|
||||
let list_stream = self.client.list(None).await
|
||||
.map_err(|e| StorageError::S3Error {
|
||||
message: format!("Failed to list objects for statistics: {}", e),
|
||||
})?;
|
||||
|
||||
use futures::TryStreamExt;
|
||||
let mut stream = list_stream;
|
||||
|
||||
while let Some(meta) = stream.try_next().await
|
||||
.map_err(|e| StorageError::S3Error {
|
||||
message: format!("Failed to read archival statistics: {}", e),
|
||||
})?
|
||||
{
|
||||
total_objects += 1;
|
||||
total_size += meta.size;
|
||||
|
||||
// Categorize by data type (first part of path)
|
||||
let key = meta.location.as_ref();
|
||||
let data_type = key.split('/').next().unwrap_or("unknown").to_string();
|
||||
*storage_by_type.entry(data_type).or_insert(0) += meta.size;
|
||||
|
||||
// Try to get compression ratio from metadata (if available)
|
||||
if let (Some(original_size), Some(compressed_size)) = (
|
||||
meta.metadata.get("original_size").and_then(|s| s.parse::<u64>().ok()),
|
||||
meta.metadata.get("compressed_size").and_then(|s| s.parse::<u64>().ok())
|
||||
) {
|
||||
if original_size > 0 {
|
||||
compression_ratios.push(compressed_size as f64 / original_size as f64);
|
||||
}
|
||||
}
|
||||
|
||||
if response.is_truncated.unwrap_or(false) {
|
||||
continuation_token = response.next_continuation_token;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
let average_compression_ratio = if !compression_ratios.is_empty() {
|
||||
compression_ratios.iter().sum::<f64>() / compression_ratios.len() as f64
|
||||
} else {
|
||||
1.0
|
||||
};
|
||||
|
||||
|
||||
// Rough cost estimation (simplified - actual costs vary by region and storage class)
|
||||
let estimated_monthly_cost = (total_size as f64 / 1_073_741_824.0) * 0.023; // ~$0.023/GB for Standard-IA
|
||||
|
||||
|
||||
Ok(ArchivalStats {
|
||||
total_objects,
|
||||
total_size_bytes: total_size,
|
||||
@@ -960,9 +807,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_archival_data_type_storage_class() {
|
||||
assert_eq!(ArchivalDataType::MarketData.storage_class(), StorageClass::StandardIa);
|
||||
assert_eq!(ArchivalDataType::TradingLogs.storage_class(), StorageClass::Standard);
|
||||
assert_eq!(ArchivalDataType::ComplianceData.storage_class(), StorageClass::Glacier);
|
||||
assert_eq!(ArchivalDataType::MarketData.storage_class_name(), "STANDARD_IA");
|
||||
assert_eq!(ArchivalDataType::TradingLogs.storage_class_name(), "STANDARD");
|
||||
assert_eq!(ArchivalDataType::ComplianceData.storage_class_name(), "GLACIER");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -261,7 +261,7 @@ impl ConfigManagerDashboard {
|
||||
|
||||
match client.get_configuration(request).await {
|
||||
Ok(response) => {
|
||||
let configs = self.convert_proto_to_config_values(response.into_inner(), category.clone());
|
||||
let configs = Self::convert_proto_to_config_values(response.into_inner(), category.clone());
|
||||
if let Some(dashboard) = self.category_dashboards.get_mut(&category) {
|
||||
dashboard.update(configs)?;
|
||||
}
|
||||
@@ -276,10 +276,10 @@ impl ConfigManagerDashboard {
|
||||
}
|
||||
|
||||
/// Convert proto ConfigResponse to our ConfigValue types
|
||||
fn convert_proto_to_config_values(&self, response: ConfigResponse, category: ConfigCategory) -> Vec<ConfigValue> {
|
||||
fn convert_proto_to_config_values(response: ConfigResponse, category: ConfigCategory) -> Vec<ConfigValue> {
|
||||
response.settings.into_iter().map(|setting| {
|
||||
ConfigValue {
|
||||
key: setting.key,
|
||||
key: setting.key.clone(),
|
||||
value: serde_json::from_str(&setting.value).unwrap_or_else(|_| {
|
||||
serde_json::Value::String(setting.value)
|
||||
}),
|
||||
@@ -343,35 +343,54 @@ impl ConfigManagerDashboard {
|
||||
async fn process_config_changes(&mut self) -> Result<()> {
|
||||
if let Some(ref mut change_rx) = self.change_receiver {
|
||||
while let Ok(change) = change_rx.try_recv() {
|
||||
info!("Configuration change received: {}.{}",
|
||||
change.category.table_name(), change.key);
|
||||
|
||||
// Add to recent changes for audit trail
|
||||
self.recent_changes.push(change.clone());
|
||||
if self.recent_changes.len() > 100 {
|
||||
self.recent_changes.remove(0);
|
||||
}
|
||||
|
||||
// Reload affected category
|
||||
if let Some(config_manager) = &self.config_manager {
|
||||
match config_manager.get_category_configs(change.category.clone()).await {
|
||||
Ok(configs) => {
|
||||
if let Some(dashboard) = self.category_dashboards.get_mut(&change.category) {
|
||||
dashboard.update(configs)?;
|
||||
if let Some(setting) = &change.setting {
|
||||
info!("Configuration change received: {}.{}",
|
||||
setting.category, setting.key);
|
||||
|
||||
// Add to recent changes for audit trail
|
||||
self.recent_changes.push(change.clone());
|
||||
if self.recent_changes.len() > 100 {
|
||||
self.recent_changes.remove(0);
|
||||
}
|
||||
|
||||
// Determine category and reload
|
||||
let category = match setting.category.as_str() {
|
||||
"trading" => ConfigCategory::Trading,
|
||||
"risk" => ConfigCategory::Risk,
|
||||
"ml" => ConfigCategory::MachineLearning,
|
||||
"security" => ConfigCategory::Security,
|
||||
"performance" => ConfigCategory::Performance,
|
||||
_ => ConfigCategory::Trading, // Default fallback
|
||||
};
|
||||
|
||||
// Reload affected category via gRPC
|
||||
if let Some(ref mut client) = self.config_client {
|
||||
let request = crate::proto::config::ConfigRequest {
|
||||
keys: vec![],
|
||||
category: Some(setting.category.clone()),
|
||||
environment: None,
|
||||
include_sensitive: false,
|
||||
};
|
||||
|
||||
match client.get_configuration(request).await {
|
||||
Ok(response) => {
|
||||
let configs = Self::convert_proto_to_config_values(response.into_inner(), category.clone());
|
||||
if let Some(dashboard) = self.category_dashboards.get_mut(&category) {
|
||||
dashboard.update(configs)?;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to reload category {:?} after change: {}", category, e);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to reload category {:?} after change: {}", change.category, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
self.needs_redraw = true;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}}
|
||||
|
||||
impl Dashboard for ConfigManagerDashboard {
|
||||
fn render(&mut self, frame: &mut Frame, area: Rect) -> Result<()> {
|
||||
@@ -471,8 +490,8 @@ impl Dashboard for ConfigManagerDashboard {
|
||||
Ok(None)
|
||||
}
|
||||
KeyCode::F(5) => {
|
||||
// Refresh all configurations
|
||||
if let Some(config_manager) = &self.config_manager {
|
||||
// Refresh all configurations via gRPC
|
||||
if self.config_client.is_some() {
|
||||
let event_sender = self.event_sender.clone();
|
||||
tokio::spawn(async move {
|
||||
let _ = event_sender.send(DashboardEvent::RefreshConfig).await;
|
||||
@@ -481,8 +500,14 @@ impl Dashboard for ConfigManagerDashboard {
|
||||
Ok(None)
|
||||
}
|
||||
KeyCode::Char('h') => {
|
||||
// Show health status
|
||||
self.show_success(format!("Health Status: {:?}", self.health_status));
|
||||
// Show connection status
|
||||
let status_msg = match &self.connection_status {
|
||||
ConnectionStatus::Connected { postgres, vault } => {
|
||||
format!("Connected - PostgreSQL: {}, Vault: {}", postgres, vault)
|
||||
}
|
||||
status => format!("Status: {:?}", status)
|
||||
};
|
||||
self.show_success(status_msg);
|
||||
Ok(None)
|
||||
}
|
||||
KeyCode::Esc | KeyCode::Char('q') => Ok(Some(DashboardEvent::Exit)),
|
||||
@@ -510,7 +535,7 @@ impl Dashboard for ConfigManagerDashboard {
|
||||
fn update(&mut self, event: DashboardEvent) -> Result<()> {
|
||||
match event {
|
||||
DashboardEvent::RefreshConfig => {
|
||||
if let Some(config_manager) = &self.config_manager {
|
||||
if self.config_client.is_some() {
|
||||
let event_sender = self.event_sender.clone();
|
||||
tokio::spawn(async move {
|
||||
// Refresh configurations in background
|
||||
@@ -521,11 +546,8 @@ impl Dashboard for ConfigManagerDashboard {
|
||||
}
|
||||
DashboardEvent::ConfigChanged { category, key } => {
|
||||
info!("Configuration changed: {}.{}", category, key);
|
||||
// Reload specific category
|
||||
if let Some(_config_manager) = &self.config_manager {
|
||||
// This would be implemented as an async reload
|
||||
self.needs_redraw = true;
|
||||
}
|
||||
// Mark for redraw - actual reload happens via gRPC streaming
|
||||
self.needs_redraw = true;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user